summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMark Shannon <mark@hotpy.org>2023-12-07 12:49:40 (GMT)
committerGitHub <noreply@github.com>2023-12-07 12:49:40 (GMT)
commitb449415b2f1b41e1c44cb453428657fdf6ff1d36 (patch)
treea31422e8d6a348baf55421d40dce503d1265b8d5
parent3d712a9f4c9f366edbe16b804ec4f6ae50b0a59f (diff)
downloadcpython-b449415b2f1b41e1c44cb453428657fdf6ff1d36.zip
cpython-b449415b2f1b41e1c44cb453428657fdf6ff1d36.tar.gz
cpython-b449415b2f1b41e1c44cb453428657fdf6ff1d36.tar.bz2
GH-111485: Separate out parsing, analysis and code-gen phases of tier 1 code generator (GH-112299)
Diffstat
-rw-r--r--Include/internal/pycore_opcode_metadata.h6
-rw-r--r--Makefile.pre.in3
-rw-r--r--Python/abstract_interp_cases.c.h3
-rw-r--r--Python/bytecodes.c5
-rw-r--r--Python/executor_cases.c.h2
-rw-r--r--Python/generated_cases.c.h1047
-rw-r--r--Tools/cases_generator/analyzer.py456
-rw-r--r--Tools/cases_generator/cwriter.py111
-rw-r--r--Tools/cases_generator/generate_cases.py1
-rw-r--r--Tools/cases_generator/lexer.py9
-rw-r--r--Tools/cases_generator/mypy.ini2
-rw-r--r--Tools/cases_generator/parser.py55
-rw-r--r--Tools/cases_generator/parsing.py3
-rw-r--r--Tools/cases_generator/stack.py81
-rw-r--r--Tools/cases_generator/tier1_generator.py417
15 files changed, 1675 insertions, 526 deletions
diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h
index 4ae15e7..774c0f9 100644
--- a/Include/internal/pycore_opcode_metadata.h
+++ b/Include/internal/pycore_opcode_metadata.h
@@ -1285,11 +1285,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) {
case _INIT_CALL_PY_EXACT_ARGS:
return 1;
case _PUSH_FRAME:
- return 1;
+ return 0;
case CALL_BOUND_METHOD_EXACT_ARGS:
- return 1;
+ return 0;
case CALL_PY_EXACT_ARGS:
- return 1;
+ return 0;
case CALL_PY_WITH_DEFAULTS:
return 1;
case CALL_TYPE_1:
diff --git a/Makefile.pre.in b/Makefile.pre.in
index b5edb4e..6ac68d5 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1587,7 +1587,6 @@ regen-cases:
$(PYTHON_FOR_REGEN) \
$(srcdir)/Tools/cases_generator/generate_cases.py \
$(CASESFLAG) \
- -o $(srcdir)/Python/generated_cases.c.h.new \
-n $(srcdir)/Include/opcode_ids.h.new \
-t $(srcdir)/Python/opcode_targets.h.new \
-m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \
@@ -1595,6 +1594,8 @@ regen-cases:
-p $(srcdir)/Lib/_opcode_metadata.py.new \
-a $(srcdir)/Python/abstract_interp_cases.c.h.new \
$(srcdir)/Python/bytecodes.c
+ $(PYTHON_FOR_REGEN) \
+ $(srcdir)/Tools/cases_generator/tier1_generator.py -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c
$(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new
$(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new
$(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new
diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h
index 0d7fbe8..96ac0aa 100644
--- a/Python/abstract_interp_cases.c.h
+++ b/Python/abstract_interp_cases.c.h
@@ -774,7 +774,8 @@
}
case _PUSH_FRAME: {
- PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true);
+ STACK_SHRINK(1);
+ PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true);
break;
}
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 2075c19..bcad8dc 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -800,11 +800,11 @@ dummy_func(
// We also push it onto the stack on exit, but that's a
// different frame, and it's accounted for by _PUSH_FRAME.
op(_POP_FRAME, (retval --)) {
- assert(EMPTY());
#if TIER_ONE
assert(frame != &entry_frame);
#endif
STORE_SP();
+ assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
@@ -1165,7 +1165,6 @@ dummy_func(
}
}
-
inst(STORE_NAME, (v -- )) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *ns = LOCALS();
@@ -3130,7 +3129,7 @@ dummy_func(
// The 'unused' output effect represents the return value
// (which will be pushed when the frame returns).
// It is needed so CALL_PY_EXACT_ARGS matches its family.
- op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) {
+ op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused if (0))) {
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 547be6f..974e3f2 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -661,11 +661,11 @@
PyObject *retval;
retval = stack_pointer[-1];
STACK_SHRINK(1);
- assert(EMPTY());
#if TIER_ONE
assert(frame != &entry_frame);
#endif
STORE_SP();
+ assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 0ac99e7..24243ecf 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -1,6 +1,6 @@
-// This file is generated by Tools/cases_generator/generate_cases.py
+// This file is generated by Tools/cases_generator/tier1_generator.py
// from:
-// Python/bytecodes.c
+// ['./Python/bytecodes.c']
// Do not edit!
#ifdef TIER_TWO
@@ -8,6 +8,7 @@
#endif
#define TIER_ONE 1
+
TARGET(BEFORE_ASYNC_WITH) {
frame->instr_ptr = next_instr;
next_instr += 1;
@@ -45,9 +46,9 @@
Py_DECREF(exit);
if (true) goto pop_1_error;
}
- STACK_GROW(1);
- stack_pointer[-2] = exit;
- stack_pointer[-1] = res;
+ stack_pointer[-1] = exit;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
DISPATCH();
}
@@ -91,9 +92,9 @@
Py_DECREF(exit);
if (true) goto pop_1_error;
}
- STACK_GROW(1);
- stack_pointer[-2] = exit;
- stack_pointer[-1] = res;
+ stack_pointer[-1] = exit;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
DISPATCH();
}
@@ -103,7 +104,6 @@
INSTRUCTION_STATS(BINARY_OP);
PREDICTED(BINARY_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
- static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *rhs;
PyObject *lhs;
PyObject *res;
@@ -133,8 +133,8 @@
Py_DECREF(rhs);
if (res == NULL) goto pop_2_error;
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -142,6 +142,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -156,12 +157,12 @@
{
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval +
- ((PyFloatObject *)right)->ob_fval;
+ ((PyFloatObject *)left)->ob_fval +
+ ((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -169,6 +170,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -187,8 +189,8 @@
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -196,6 +198,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -214,8 +217,8 @@
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (res == NULL) goto pop_2_error;
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -223,6 +226,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
// _GUARD_BOTH_UNICODE
@@ -258,7 +262,7 @@
assert(next_instr->op.code == STORE_FAST);
SKIP_OVER(1);
}
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -266,6 +270,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -280,12 +285,12 @@
{
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval *
- ((PyFloatObject *)right)->ob_fval;
+ ((PyFloatObject *)left)->ob_fval *
+ ((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -293,6 +298,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -311,8 +317,8 @@
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -320,6 +326,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -334,12 +341,12 @@
{
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval -
- ((PyFloatObject *)right)->ob_fval;
+ ((PyFloatObject *)left)->ob_fval -
+ ((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -347,6 +354,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -365,8 +373,8 @@
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -393,8 +401,8 @@
}
Py_DECREF(container);
if (res == NULL) goto pop_3_error;
- STACK_SHRINK(2);
- stack_pointer[-1] = res;
+ stack_pointer[-3] = res;
+ stack_pointer += -2;
DISPATCH();
}
@@ -404,7 +412,6 @@
INSTRUCTION_STATS(BINARY_SUBSCR);
PREDICTED(BINARY_SUBSCR);
_Py_CODEUNIT *this_instr = next_instr - 2;
- static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *container;
PyObject *res;
@@ -431,8 +438,8 @@
Py_DECREF(sub);
if (res == NULL) goto pop_2_error;
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -440,6 +447,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *dict;
PyObject *res;
@@ -454,8 +462,9 @@
Py_DECREF(dict);
Py_DECREF(sub);
if (rc <= 0) goto pop_2_error;
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ // not found or error
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -463,6 +472,7 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *container;
sub = stack_pointer[-1];
@@ -494,6 +504,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *list;
PyObject *res;
@@ -501,7 +512,6 @@
list = stack_pointer[-2];
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
-
// Deopt unless 0 <= sub < PyList_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
@@ -512,8 +522,8 @@
Py_INCREF(res);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -521,6 +531,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *str;
PyObject *res;
@@ -538,8 +549,8 @@
res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(str);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -547,6 +558,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *tuple;
PyObject *res;
@@ -554,7 +566,6 @@
tuple = stack_pointer[-2];
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
-
// Deopt unless 0 <= sub < PyTuple_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
@@ -565,8 +576,8 @@
Py_INCREF(res);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(tuple);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -578,7 +589,7 @@
PyObject **values;
PyObject *map;
keys = stack_pointer[-1];
- values = stack_pointer - 1 - oparg;
+ values = &stack_pointer[-1 - oparg];
if (!PyTuple_CheckExact(keys) ||
PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
_PyErr_SetString(tstate, PyExc_SystemError,
@@ -586,15 +597,15 @@
GOTO_ERROR(error); // Pop the keys and values.
}
map = _PyDict_FromItems(
- &PyTuple_GET_ITEM(keys, 0), 1,
- values, 1, oparg);
+ &PyTuple_GET_ITEM(keys, 0), 1,
+ values, 1, oparg);
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(values[_i]);
}
Py_DECREF(keys);
- if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; }
- STACK_SHRINK(oparg);
- stack_pointer[-1] = map;
+ if (map == NULL) { stack_pointer += -1 - oparg; goto error; }
+ stack_pointer[-1 - oparg] = map;
+ stack_pointer += -oparg;
DISPATCH();
}
@@ -604,12 +615,11 @@
INSTRUCTION_STATS(BUILD_LIST);
PyObject **values;
PyObject *list;
- values = stack_pointer - oparg;
+ values = &stack_pointer[-oparg];
list = _PyList_FromArraySteal(values, oparg);
- if (list == NULL) { STACK_SHRINK(oparg); goto error; }
- STACK_SHRINK(oparg);
- STACK_GROW(1);
- stack_pointer[-1] = list;
+ if (list == NULL) { stack_pointer += -oparg; goto error; }
+ stack_pointer[-oparg] = list;
+ stack_pointer += 1 - oparg;
DISPATCH();
}
@@ -619,18 +629,17 @@
INSTRUCTION_STATS(BUILD_MAP);
PyObject **values;
PyObject *map;
- values = stack_pointer - oparg*2;
+ values = &stack_pointer[-oparg*2];
map = _PyDict_FromItems(
- values, 2,
- values+1, 2,
- oparg);
+ values, 2,
+ values+1, 2,
+ oparg);
for (int _i = oparg*2; --_i >= 0;) {
Py_DECREF(values[_i]);
}
- if (map == NULL) { STACK_SHRINK(oparg*2); goto error; }
- STACK_SHRINK(oparg*2);
- STACK_GROW(1);
- stack_pointer[-1] = map;
+ if (map == NULL) { stack_pointer += -oparg*2; goto error; }
+ stack_pointer[-oparg*2] = map;
+ stack_pointer += 1 - oparg*2;
DISPATCH();
}
@@ -640,24 +649,23 @@
INSTRUCTION_STATS(BUILD_SET);
PyObject **values;
PyObject *set;
- values = stack_pointer - oparg;
+ values = &stack_pointer[-oparg];
set = PySet_New(NULL);
if (set == NULL)
- GOTO_ERROR(error);
+ GOTO_ERROR(error);
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
if (err == 0)
- err = PySet_Add(set, item);
+ err = PySet_Add(set, item);
Py_DECREF(item);
}
if (err != 0) {
Py_DECREF(set);
- if (true) { STACK_SHRINK(oparg); goto error; }
+ if (true) { stack_pointer += -oparg; goto error; }
}
- STACK_SHRINK(oparg);
- STACK_GROW(1);
- stack_pointer[-1] = set;
+ stack_pointer[-oparg] = set;
+ stack_pointer += 1 - oparg;
DISPATCH();
}
@@ -669,17 +677,16 @@
PyObject *stop;
PyObject *start;
PyObject *slice;
- if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; }
- stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)];
- start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)];
+ if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; }
+ stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))];
+ start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))];
slice = PySlice_New(start, stop, step);
Py_DECREF(start);
Py_DECREF(stop);
Py_XDECREF(step);
- if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; }
- STACK_SHRINK(((oparg == 3) ? 1 : 0));
- STACK_SHRINK(1);
- stack_pointer[-1] = slice;
+ if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; }
+ stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice;
+ stack_pointer += -1 - (((oparg == 3) ? 1 : 0));
DISPATCH();
}
@@ -689,15 +696,14 @@
INSTRUCTION_STATS(BUILD_STRING);
PyObject **pieces;
PyObject *str;
- pieces = stack_pointer - oparg;
+ pieces = &stack_pointer[-oparg];
str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(pieces[_i]);
}
- if (str == NULL) { STACK_SHRINK(oparg); goto error; }
- STACK_SHRINK(oparg);
- STACK_GROW(1);
- stack_pointer[-1] = str;
+ if (str == NULL) { stack_pointer += -oparg; goto error; }
+ stack_pointer[-oparg] = str;
+ stack_pointer += 1 - oparg;
DISPATCH();
}
@@ -707,12 +713,11 @@
INSTRUCTION_STATS(BUILD_TUPLE);
PyObject **values;
PyObject *tup;
- values = stack_pointer - oparg;
+ values = &stack_pointer[-oparg];
tup = _PyTuple_FromArraySteal(values, oparg);
- if (tup == NULL) { STACK_SHRINK(oparg); goto error; }
- STACK_SHRINK(oparg);
- STACK_GROW(1);
- stack_pointer[-1] = tup;
+ if (tup == NULL) { stack_pointer += -oparg; goto error; }
+ stack_pointer[-oparg] = tup;
+ stack_pointer += 1 - oparg;
DISPATCH();
}
@@ -730,13 +735,12 @@
INSTRUCTION_STATS(CALL);
PREDICTED(CALL);
_Py_CODEUNIT *this_instr = next_instr - 4;
- static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
// _SPECIALIZE_CALL
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
@@ -793,12 +797,12 @@
}
/* Callable is not a normal Python function */
res = PyObject_Vectorcall(
- callable, args,
- total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- NULL);
+ callable, args,
+ total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+ NULL);
if (opcode == INSTRUMENTED_CALL) {
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : args[0];
+ &_PyInstrumentation_MISSING : args[0];
if (res == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
@@ -818,11 +822,10 @@
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -831,10 +834,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *null;
PyObject *callable;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* This instruction does the following:
@@ -890,13 +894,15 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *null;
PyObject *callable;
+ PyObject *func;
PyObject *self;
PyObject *self_or_null;
- PyObject *func;
PyObject **args;
_PyInterpreterFrame *new_frame;
+ /* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
@@ -936,7 +942,8 @@
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
}
// _INIT_CALL_PY_EXACT_ARGS
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
+ self_or_null = stack_pointer[-1 - oparg];
{
int argcount = oparg;
if (self_or_null != NULL) {
@@ -960,26 +967,26 @@
#endif
}
// _PUSH_FRAME
- STACK_SHRINK(oparg);
- STACK_SHRINK(2);
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
- STORE_SP();
+ stack_pointer += -2 - oparg;
+ _PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
- #if LLTRACE && TIER_ONE
+ #if LLTRACE && TIER_ONE
lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
if (lltrace < 0) {
goto exit_unwind;
}
- #endif
+ #endif
}
+ stack_pointer += (((0) ? 1 : 0));
DISPATCH();
}
@@ -987,11 +994,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_CLASS);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int total_args = oparg;
@@ -1009,10 +1017,9 @@
Py_DECREF(args[i]);
}
Py_DECREF(tp);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1021,11 +1028,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_FASTCALL functions, without keywords */
@@ -1044,21 +1052,19 @@
args,
total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- /* Not deopting because this doesn't mean our optimization was
- wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
- 'invalid'). In those cases an exception is set, so we must
- handle it.
- */
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ /* Not deopting because this doesn't mean our optimization was
+ wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
+ 'invalid'). In those cases an exception is set, so we must
+ handle it.
+ */
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1067,11 +1073,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
@@ -1085,20 +1092,18 @@
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =
- (_PyCFunctionFastWithKeywords)(void(*)(void))
- PyCFunction_GET_FUNCTION(callable);
+ (_PyCFunctionFastWithKeywords)(void(*)(void))
+ PyCFunction_GET_FUNCTION(callable);
res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1107,11 +1112,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_O);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_O functions */
@@ -1134,13 +1140,11 @@
res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-
Py_DECREF(arg);
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1155,9 +1159,9 @@
PyObject *callargs;
PyObject *func;
PyObject *result;
- if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; }
- callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)];
- func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)];
+ if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; }
+ callargs = stack_pointer[-1 - ((oparg & 1))];
+ func = stack_pointer[-3 - ((oparg & 1))];
// DICT_MERGE is called before this opcode if there are kwargs.
// It converts all dict subtypes in kwargs into regular dicts.
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
@@ -1177,7 +1181,7 @@
!PyFunction_Check(func) && !PyMethod_Check(func)
) {
PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ?
- PyTuple_GET_ITEM(callargs, 0) : Py_None;
+ PyTuple_GET_ITEM(callargs, 0) : Py_None;
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
@@ -1205,10 +1209,9 @@
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
-
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
- (PyFunctionObject *)func, locals,
- nargs, callargs, kwargs);
+ (PyFunctionObject *)func, locals,
+ nargs, callargs, kwargs);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
@@ -1224,10 +1227,9 @@
Py_DECREF(callargs);
Py_XDECREF(kwargs);
assert(PEEK(2 + (oparg & 1)) == NULL);
- if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; }
- STACK_SHRINK(((oparg & 1) ? 1 : 0));
- STACK_SHRINK(2);
- stack_pointer[-1] = result;
+ if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; }
+ stack_pointer[-3 - ((oparg & 1))] = result;
+ stack_pointer += -2 - ((oparg & 1));
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1261,8 +1263,8 @@
Py_DECREF(value2);
Py_DECREF(value1);
if (res == NULL) goto pop_2_error;
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -1270,11 +1272,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ISINSTANCE);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* isinstance(o, o2) */
@@ -1295,14 +1298,12 @@
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
DISPATCH();
}
@@ -1318,7 +1319,7 @@
PyObject *callable;
PyObject *res;
kwnames = stack_pointer[-1];
- args = stack_pointer - 1 - oparg;
+ args = &stack_pointer[-1 - oparg];
self_or_null = stack_pointer[-2 - oparg];
callable = stack_pointer[-3 - oparg];
// oparg counts all of the args, but *not* self:
@@ -1363,12 +1364,12 @@
}
/* Callable is not a normal Python function */
res = PyObject_Vectorcall(
- callable, args,
- positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- kwnames);
+ callable, args,
+ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+ kwnames);
if (opcode == INSTRUMENTED_CALL_KW) {
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : args[0];
+ &_PyInstrumentation_MISSING : args[0];
if (res == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
@@ -1389,10 +1390,9 @@
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_3_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(2);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -3 - oparg; goto error; }
+ stack_pointer[-3 - oparg] = res;
+ stack_pointer += -2 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1401,11 +1401,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LEN);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* len(o) */
@@ -1425,13 +1426,11 @@
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-
Py_DECREF(callable);
Py_DECREF(arg);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
DISPATCH();
}
@@ -1439,10 +1438,11 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LIST_APPEND);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self;
PyObject *callable;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
assert(oparg == 1);
@@ -1467,11 +1467,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int total_args = oparg;
@@ -1488,7 +1489,7 @@
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
STAT_INC(CALL, hit);
_PyCFunctionFast cfunc =
- (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
+ (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
res = cfunc(self, args + 1, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -1497,10 +1498,9 @@
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1509,11 +1509,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int total_args = oparg;
@@ -1531,19 +1532,17 @@
STAT_INC(CALL, hit);
int nargs = total_args - 1;
_PyCFunctionFastWithKeywords cfunc =
- (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
+ (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
res = cfunc(self, args + 1, nargs, NULL);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1552,11 +1551,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
assert(oparg == 0 || oparg == 1);
@@ -1584,10 +1584,9 @@
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1596,11 +1595,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int total_args = oparg;
@@ -1629,10 +1629,9 @@
Py_DECREF(self);
Py_DECREF(arg);
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1641,10 +1640,12 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_EXACT_ARGS);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *self_or_null;
PyObject *callable;
PyObject **args;
_PyInterpreterFrame *new_frame;
+ /* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
@@ -1668,7 +1669,8 @@
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
}
// _INIT_CALL_PY_EXACT_ARGS
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
+ self_or_null = stack_pointer[-1 - oparg];
{
int argcount = oparg;
if (self_or_null != NULL) {
@@ -1692,26 +1694,26 @@
#endif
}
// _PUSH_FRAME
- STACK_SHRINK(oparg);
- STACK_SHRINK(2);
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
- STORE_SP();
+ stack_pointer += -2 - oparg;
+ _PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
- #if LLTRACE && TIER_ONE
+ #if LLTRACE && TIER_ONE
lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
if (lltrace < 0) {
goto exit_unwind;
}
- #endif
+ #endif
}
+ stack_pointer += (((0) ? 1 : 0));
DISPATCH();
}
@@ -1719,10 +1721,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
@@ -1763,11 +1766,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_STR_1);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
assert(oparg == 1);
@@ -1778,10 +1782,9 @@
res = PyObject_Str(arg);
Py_DECREF(arg);
Py_DECREF(&PyUnicode_Type); // I.e., callable
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1790,11 +1793,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TUPLE_1);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
assert(oparg == 1);
@@ -1805,10 +1809,9 @@
res = PySequence_Tuple(arg);
Py_DECREF(arg);
Py_DECREF(&PyTuple_Type); // I.e., tuple
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
@@ -1817,11 +1820,12 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TYPE_1);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *null;
PyObject *callable;
PyObject *res;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
assert(oparg == 1);
@@ -1832,9 +1836,8 @@
res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(obj);
Py_DECREF(&PyType_Type); // I.e., callable
- STACK_SHRINK(oparg);
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2 - oparg] = res;
+ stack_pointer += -1 - oparg;
DISPATCH();
}
@@ -1853,18 +1856,15 @@
Py_DECREF(match_type);
if (true) goto pop_2_error;
}
-
match = NULL;
rest = NULL;
int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
- &match, &rest);
+ &match, &rest);
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (res < 0) goto pop_2_error;
-
assert((match == NULL) == (rest == NULL));
if (match == NULL) goto pop_2_error;
-
if (!Py_IsNone(match)) {
PyErr_SetHandledException(match);
}
@@ -1884,10 +1884,9 @@
left = stack_pointer[-2];
assert(PyExceptionInstance_Check(left));
if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
- Py_DECREF(right);
- if (true) goto pop_1_error;
+ Py_DECREF(right);
+ if (true) goto pop_1_error;
}
-
int res = PyErr_GivenExceptionMatches(left, right);
Py_DECREF(right);
b = res ? Py_True : Py_False;
@@ -1922,9 +1921,9 @@
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
- STACK_SHRINK(1);
- stack_pointer[-2] = none;
- stack_pointer[-1] = value;
+ stack_pointer[-3] = none;
+ stack_pointer[-2] = value;
+ stack_pointer += -1;
DISPATCH();
}
@@ -1934,7 +1933,6 @@
INSTRUCTION_STATS(COMPARE_OP);
PREDICTED(COMPARE_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
- static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -1968,8 +1966,8 @@
res = res_bool ? Py_True : Py_False;
}
}
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -1977,6 +1975,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_FLOAT);
+ static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -1993,8 +1992,8 @@
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
res = (sign_ish & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -2002,6 +2001,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_INT);
+ static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -2022,8 +2022,8 @@
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
res = (sign_ish & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -2031,6 +2031,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_STR);
+ static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
@@ -2048,8 +2049,8 @@
assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -2067,8 +2068,8 @@
Py_DECREF(right);
if (res < 0) goto pop_2_error;
b = (res ^ oparg) ? Py_True : Py_False;
- STACK_SHRINK(1);
- stack_pointer[-1] = b;
+ stack_pointer[-2] = b;
+ stack_pointer += -1;
DISPATCH();
}
@@ -2098,8 +2099,8 @@
bottom = stack_pointer[-1 - (oparg-1)];
assert(oparg > 0);
top = Py_NewRef(bottom);
- STACK_GROW(1);
- stack_pointer[-1] = top;
+ stack_pointer[0] = top;
+ stack_pointer += 1;
DISPATCH();
}
@@ -2130,7 +2131,7 @@
int err = PyObject_DelAttr(owner, name);
Py_DECREF(owner);
if (err) goto pop_1_error;
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -2172,7 +2173,7 @@
if (err != 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ NAME_ERROR_MSG, name);
}
GOTO_ERROR(error);
}
@@ -2195,8 +2196,8 @@
// Can't use ERROR_IF here.
if (err != 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
- NAME_ERROR_MSG,
- name);
+ NAME_ERROR_MSG,
+ name);
GOTO_ERROR(error);
}
DISPATCH();
@@ -2215,7 +2216,7 @@
Py_DECREF(container);
Py_DECREF(sub);
if (err) goto pop_2_error;
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -2235,7 +2236,7 @@
if (true) goto pop_1_error;
}
Py_DECREF(update);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -2250,14 +2251,14 @@
if (PyDict_Update(dict, update) < 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
_PyErr_Format(tstate, PyExc_TypeError,
- "'%.200s' object is not a mapping",
- Py_TYPE(update)->tp_name);
+ "'%.200s' object is not a mapping",
+ Py_TYPE(update)->tp_name);
}
Py_DECREF(update);
if (true) goto pop_1_error;
}
Py_DECREF(update);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -2281,7 +2282,7 @@
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -2290,17 +2291,17 @@
next_instr += 1;
INSTRUCTION_STATS(END_FOR);
PyObject *value;
- // POP_TOP
+ // _POP_TOP
value = stack_pointer[-1];
{
Py_DECREF(value);
}
- // POP_TOP
+ // _POP_TOP
value = stack_pointer[-2];
{
Py_DECREF(value);
}
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -2313,8 +2314,8 @@
value = stack_pointer[-1];
receiver = stack_pointer[-2];
Py_DECREF(receiver);
- STACK_SHRINK(1);
- stack_pointer[-1] = value;
+ stack_pointer[-2] = value;
+ stack_pointer += -1;
DISPATCH();
}
@@ -2324,7 +2325,6 @@
INSTRUCTION_STATS(ENTER_EXECUTOR);
TIER_ONE_ONLY
CHECK_EVAL_BREAKER();
-
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255];
int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00);
@@ -2352,11 +2352,11 @@
assert(STACK_LEVEL() == 2);
if (should_be_none != Py_None) {
PyErr_Format(PyExc_TypeError,
- "__init__() should return None, not '%.200s'",
- Py_TYPE(should_be_none)->tp_name);
+ "__init__() should return None, not '%.200s'",
+ Py_TYPE(should_be_none)->tp_name);
GOTO_ERROR(error);
}
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -2405,8 +2405,8 @@
Py_DECREF(value);
Py_DECREF(fmt_spec);
if (res == NULL) goto pop_2_error;
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -2416,7 +2416,6 @@
INSTRUCTION_STATS(FOR_ITER);
PREDICTED(FOR_ITER);
_Py_CODEUNIT *this_instr = next_instr - 2;
- static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
PyObject *next;
// _SPECIALIZE_FOR_ITER
@@ -2448,7 +2447,7 @@
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
- next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
+ next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
Py_DECREF(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR instruction */
@@ -2457,8 +2456,8 @@
}
// Common case: no jump, leave it to the code generator
}
- STACK_GROW(1);
- stack_pointer[-1] = next;
+ stack_pointer[0] = next;
+ stack_pointer += 1;
DISPATCH();
}
@@ -2466,6 +2465,7 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_GEN);
+ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
iter = stack_pointer[-1];
DEOPT_IF(tstate->interp->eval_frame, FOR_ITER);
@@ -2489,8 +2489,10 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_LIST);
+ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
PyObject *next;
+ /* Skip 1 cache entry */
// _ITER_CHECK_LIST
iter = stack_pointer[-1];
{
@@ -2523,8 +2525,8 @@
assert(it->it_index < PyList_GET_SIZE(seq));
next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++));
}
- STACK_GROW(1);
- stack_pointer[-1] = next;
+ stack_pointer[0] = next;
+ stack_pointer += 1;
DISPATCH();
}
@@ -2532,8 +2534,10 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_RANGE);
+ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
PyObject *next;
+ /* Skip 1 cache entry */
// _ITER_CHECK_RANGE
iter = stack_pointer[-1];
{
@@ -2564,8 +2568,8 @@
next = PyLong_FromLong(value);
if (next == NULL) goto error;
}
- STACK_GROW(1);
- stack_pointer[-1] = next;
+ stack_pointer[0] = next;
+ stack_pointer += 1;
DISPATCH();
}
@@ -2573,8 +2577,10 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_TUPLE);
+ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
PyObject *next;
+ /* Skip 1 cache entry */
// _ITER_CHECK_TUPLE
iter = stack_pointer[-1];
{
@@ -2607,8 +2613,8 @@
assert(it->it_index < PyTuple_GET_SIZE(seq));
next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++));
}
- STACK_GROW(1);
- stack_pointer[-1] = next;
+ stack_pointer[0] = next;
+ stack_pointer += 1;
DISPATCH();
}
@@ -2621,11 +2627,9 @@
obj = stack_pointer[-1];
unaryfunc getter = NULL;
PyTypeObject *type = Py_TYPE(obj);
-
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
-
if (getter == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an object with "
@@ -2634,14 +2638,11 @@
Py_DECREF(obj);
if (true) goto pop_1_error;
}
-
iter = (*getter)(obj);
Py_DECREF(obj);
if (iter == NULL) goto pop_1_error;
-
if (Py_TYPE(iter)->tp_as_async == NULL ||
- Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
-
+ Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
@@ -2663,7 +2664,6 @@
unaryfunc getter = NULL;
PyObject *next_iter = NULL;
PyTypeObject *type = Py_TYPE(aiter);
-
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
@@ -2673,7 +2673,6 @@
if (type->tp_as_async != NULL){
getter = type->tp_as_async->am_anext;
}
-
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
@@ -2687,7 +2686,6 @@
type->tp_name);
GOTO_ERROR(error);
}
-
awaitable = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
_PyErr_FormatFromCause(
@@ -2695,15 +2693,14 @@
"'async for' received an invalid object "
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
-
Py_DECREF(next_iter);
GOTO_ERROR(error);
} else {
Py_DECREF(next_iter);
}
}
- STACK_GROW(1);
- stack_pointer[-1] = awaitable;
+ stack_pointer[0] = awaitable;
+ stack_pointer += 1;
DISPATCH();
}
@@ -2715,13 +2712,10 @@
PyObject *iter;
iterable = stack_pointer[-1];
iter = _PyCoro_GetAwaitableIter(iterable);
-
if (iter == NULL) {
_PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
}
-
Py_DECREF(iterable);
-
if (iter != NULL && PyCoro_CheckExact(iter)) {
PyObject *yf = _PyGen_yf((PyGenObject*)iter);
if (yf != NULL) {
@@ -2735,7 +2729,6 @@
/* The code below jumps to `error` if `iter` is NULL. */
}
}
-
if (iter == NULL) goto pop_1_error;
stack_pointer[-1] = iter;
DISPATCH();
@@ -2768,8 +2761,8 @@
if (len_i < 0) goto error;
len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) goto error;
- STACK_GROW(1);
- stack_pointer[-1] = len_o;
+ stack_pointer[0] = len_o;
+ stack_pointer += 1;
DISPATCH();
}
@@ -2819,8 +2812,8 @@
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
res = import_from(tstate, from, name);
if (res == NULL) goto error;
- STACK_GROW(1);
- stack_pointer[-1] = res;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
DISPATCH();
}
@@ -2839,8 +2832,8 @@
Py_DECREF(level);
Py_DECREF(fromlist);
if (res == NULL) goto pop_2_error;
- STACK_SHRINK(1);
- stack_pointer[-1] = res;
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
DISPATCH();
}
@@ -2852,10 +2845,10 @@
int total_args = oparg + is_meth;
PyObject *function = PEEK(oparg + 2);
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : PEEK(total_args);
+ &_PyInstrumentation_MISSING : PEEK(total_args);
int err = _Py_call_instrumentation_2args(
- tstate, PY_MONITORING_EVENT_CALL,
- frame, this_instr, function, arg);
+ tstate, PY_MONITORING_EVENT_CALL,
+ frame, this_instr, function, arg);
if (err) goto error;
INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
GO_TO_INSTRUCTION(CALL);
@@ -2876,10 +2869,10 @@
int total_args = oparg + is_meth;
PyObject *function = PEEK(oparg + 3);
PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING
- : PEEK(total_args + 1);
+ : PEEK(total_args + 1);
int err = _Py_call_instrumentation_2args(
- tstate, PY_MONITORING_EVENT_CALL,
- frame, this_instr, function, arg);
+ tstate, PY_MONITORING_EVENT_CALL,
+ frame, this_instr, function, arg);
if (err) goto error;
GO_TO_INSTRUCTION(CALL_KW);
}
@@ -2904,7 +2897,7 @@
}
Py_DECREF(receiver);
Py_DECREF(value);
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -2925,8 +2918,8 @@
PyErr_SetRaisedException(NULL);
}
Py_DECREF(receiver);
- STACK_SHRINK(1);
- stack_pointer[-1] = value;
+ stack_pointer[-2] = value;
+ stack_pointer += -1;
DISPATCH();
}
@@ -3094,7 +3087,7 @@
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation(
- tstate, oparg > 0, frame, this_instr);
+ tstate, oparg > 0, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) goto error;
if (frame->instr_ptr != this_instr) {
@@ -3112,8 +3105,8 @@
INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST);
PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg);
int err = _Py_call_instrumentation_arg(
- tstate, PY_MONITORING_EVENT_PY_RETURN,
- frame, this_instr, retval);
+ tstate, PY_MONITORING_EVENT_PY_RETURN,
+ frame, this_instr, retval);
if (err) GOTO_ERROR(error);
Py_INCREF(retval);
assert(EMPTY());
@@ -3136,8 +3129,8 @@
PyObject *retval;
retval = stack_pointer[-1];
int err = _Py_call_instrumentation_arg(
- tstate, PY_MONITORING_EVENT_PY_RETURN,
- frame, this_instr, retval);
+ tstate, PY_MONITORING_EVENT_PY_RETURN,
+ frame, this_instr, retval);
if (err) GOTO_ERROR(error);
STACK_SHRINK(1);
assert(EMPTY());
@@ -3167,8 +3160,8 @@
gen->gi_frame_state = FRAME_SUSPENDED + oparg;
_PyFrame_SetStackPointer(frame, stack_pointer - 1);
int err = _Py_call_instrumentation_arg(
- tstate, PY_MONITORING_EVENT_PY_YIELD,
- frame, this_instr, retval);
+ tstate, PY_MONITORING_EVENT_PY_YIELD,
+ frame, this_instr, retval);
if (err) GOTO_ERROR(error);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
@@ -3211,8 +3204,8 @@
Py_DECREF(left);
Py_DECREF(right);
b = res ? Py_True : Py_False;
- STACK_SHRINK(1);
- stack_pointer[-1] = b;
+ stack_pointer[-2] = b;
+ stack_pointer += -1;
DISPATCH();
}
@@ -3286,7 +3279,7 @@
v = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error;
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -3301,19 +3294,19 @@
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
if (none_val == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
- (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
+ (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
{
_PyErr_Clear(tstate);
_PyErr_Format(tstate, PyExc_TypeError,
- "Value after * must be an iterable, not %.200s",
- Py_TYPE(iterable)->tp_name);
+ "Value after * must be an iterable, not %.200s",
+ Py_TYPE(iterable)->tp_name);
}
Py_DECREF(iterable);
if (true) goto pop_1_error;
}
assert(Py_IsNone(none_val));
Py_DECREF(iterable);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -3323,8 +3316,8 @@
INSTRUCTION_STATS(LOAD_ASSERTION_ERROR);
PyObject *value;
value = Py_NewRef(PyExc_AssertionError);
- STACK_GROW(1);
- stack_pointer[-1] = value;
+ stack_pointer[0] = value;
+ stack_pointer += 1;
DISPATCH();
}
@@ -3334,7 +3327,6 @@
INSTRUCTION_STATS(LOAD_ATTR);
PREDICTED(LOAD_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 10;
- static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *self_or_null = NULL;
@@ -3374,7 +3366,7 @@
the second element of the stack to NULL, to signal
CALL that it's not a method call.
NULL | meth | arg1 | ... | argN
- */
+ */
Py_DECREF(owner);
if (attr == NULL) goto pop_1_error;
self_or_null = NULL;
@@ -3387,9 +3379,9 @@
if (attr == NULL) goto pop_1_error;
}
}
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; }
+ stack_pointer[-1] = attr;
+ if (oparg & 1) stack_pointer[0] = self_or_null;
+ stack_pointer += ((oparg & 1));
DISPATCH();
}
@@ -3397,9 +3389,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_CLASS);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
+ /* Skip 1 cache entry */
// _CHECK_ATTR_CLASS
owner = stack_pointer[-1];
{
@@ -3408,6 +3402,7 @@
assert(type_version != 0);
DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR);
}
+ /* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
{
PyObject *descr = read_obj(&this_instr[6].cache);
@@ -3417,9 +3412,9 @@
null = NULL;
Py_DECREF(owner);
}
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ stack_pointer[-1] = attr;
+ if (oparg & 1) stack_pointer[0] = null;
+ stack_pointer += ((oparg & 1));
DISPATCH();
}
@@ -3427,6 +3422,7 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
@@ -3445,7 +3441,6 @@
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
-
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
Py_INCREF(f);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2);
@@ -3461,9 +3456,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -3490,9 +3487,10 @@
null = NULL;
Py_DECREF(owner);
}
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ /* Skip 5 cache entries */
+ stack_pointer[-1] = attr;
+ if (oparg & 1) stack_pointer[0] = null;
+ stack_pointer += ((oparg & 1));
DISPATCH();
}
@@ -3500,9 +3498,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
- PyObject *self;
+ PyObject *self = NULL;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -3519,6 +3519,7 @@
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
+ /* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_LAZY_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
@@ -3529,9 +3530,9 @@
attr = Py_NewRef(descr);
self = owner;
}
- STACK_GROW(1);
- stack_pointer[-2] = attr;
- stack_pointer[-1] = self;
+ stack_pointer[-1] = attr;
+ if (1) stack_pointer[0] = self;
+ stack_pointer += (((1) ? 1 : 0));
DISPATCH();
}
@@ -3539,9 +3540,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
- PyObject *self;
+ PyObject *self = NULL;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -3550,6 +3553,7 @@
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
+ /* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
@@ -3561,9 +3565,9 @@
attr = Py_NewRef(descr);
self = owner;
}
- STACK_GROW(1);
- stack_pointer[-2] = attr;
- stack_pointer[-1] = self;
+ stack_pointer[-1] = attr;
+ if (1) stack_pointer[0] = self;
+ stack_pointer += (((1) ? 1 : 0));
DISPATCH();
}
@@ -3571,9 +3575,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
- PyObject *self;
+ PyObject *self = NULL;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -3606,9 +3612,9 @@
assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR));
self = owner;
}
- STACK_GROW(1);
- stack_pointer[-2] = attr;
- stack_pointer[-1] = self;
+ stack_pointer[-1] = attr;
+ if (1) stack_pointer[0] = self;
+ stack_pointer += (((1) ? 1 : 0));
DISPATCH();
}
@@ -3616,9 +3622,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_MODULE);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
+ /* Skip 1 cache entry */
// _CHECK_ATTR_MODULE
owner = stack_pointer[-1];
{
@@ -3642,9 +3650,10 @@
null = NULL;
Py_DECREF(owner);
}
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ /* Skip 5 cache entries */
+ stack_pointer[-1] = attr;
+ if (oparg & 1) stack_pointer[0] = null;
+ stack_pointer += ((oparg & 1));
DISPATCH();
}
@@ -3652,8 +3661,10 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -3662,6 +3673,7 @@
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
+ /* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
@@ -3673,6 +3685,7 @@
attr = Py_NewRef(descr);
}
stack_pointer[-1] = attr;
+ stack_pointer += (((0) ? 1 : 0));
DISPATCH();
}
@@ -3680,8 +3693,10 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -3713,6 +3728,7 @@
attr = Py_NewRef(descr);
}
stack_pointer[-1] = attr;
+ stack_pointer += (((0) ? 1 : 0));
DISPATCH();
}
@@ -3720,6 +3736,7 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_PROPERTY);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
@@ -3727,7 +3744,6 @@
PyObject *fget = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
-
PyTypeObject *cls = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
@@ -3752,9 +3768,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_SLOT);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -3774,9 +3792,10 @@
null = NULL;
Py_DECREF(owner);
}
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ /* Skip 5 cache entries */
+ stack_pointer[-1] = attr;
+ if (oparg & 1) stack_pointer[0] = null;
+ stack_pointer += ((oparg & 1));
DISPATCH();
}
@@ -3784,9 +3803,11 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -3827,9 +3848,10 @@
null = NULL;
Py_DECREF(owner);
}
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ /* Skip 5 cache entries */
+ stack_pointer[-1] = attr;
+ if (oparg & 1) stack_pointer[0] = null;
+ stack_pointer += ((oparg & 1));
DISPATCH();
}
@@ -3844,8 +3866,8 @@
"__build_class__ not found");
if (true) goto error;
}
- STACK_GROW(1);
- stack_pointer[-1] = bc;
+ stack_pointer[0] = bc;
+ stack_pointer += 1;
DISPATCH();
}
@@ -3856,8 +3878,8 @@
PyObject *value;
value = GETITEM(FRAME_CO_CONSTS, oparg);
Py_INCREF(value);
- STACK_GROW(1);
- stack_pointer[-1] = value;
+ stack_pointer[0] = value;
+ stack_pointer += 1;
DISPATCH();
}
@@ -3873,8 +3895,8 @@
if (true) goto error;
}
Py_INCREF(value);
- STACK_GROW(1);
- stack_pointer[-1] = value;
+ stack_pointer[0] = value;
+ stack_pointer += 1;
DISPATCH();
}
@@ -3886,8 +3908,8 @@
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
- STACK_GROW(1);
- stack_pointer[-1] = value;
+ stack_pointer[0] = value;
+ stack_pointer += 1;
DISPATCH();
}
@@ -3899,8 +3921,8 @@
value = GETLOCAL(oparg);
// do not use SETLOCAL here, it decrefs the old value
GETLOCAL(oparg) = NULL;
- STACK_GROW(1);
- stack_pointer[-1] = value;
+ stack_pointer[0] = value;
+ stack_pointer += 1;
DISPATCH();
}
@@ -3912,8 +3934,8 @@
value = GETLOCAL(oparg);
if (value == NULL) goto unbound_local_error;
Py_INCREF(value);
- STACK_GROW(1);
- stack_pointer[-1] = value;
+ stack_pointer[0] = value;
+ stack_pointer += 1;
DISPATCH();
}
@@ -3929,9 +3951,9 @@
value2 = GETLOCAL(oparg2);
Py_INCREF(value1);
Py_INCREF(value2);
- STACK_GROW(2);
- stack_pointer[-2] = value1;
- stack_pointer[-1] = value2;
+ stack_pointer[0] = value1;
+ stack_pointer[1] = value2;
+ stack_pointer += 2;
DISPATCH();
}
@@ -3984,8 +4006,8 @@
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
- tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
GOTO_ERROR(error);
}
}
@@ -4001,7 +4023,6 @@
INSTRUCTION_STATS(LOAD_GLOBAL);
PREDICTED(LOAD_GLOBAL);
_Py_CODEUNIT *this_instr = next_instr - 5;
- static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
PyObject *res;
PyObject *null = NULL;
// _SPECIALIZE_LOAD_GLOBAL
@@ -4026,14 +4047,14 @@
&& PyDict_CheckExact(BUILTINS()))
{
res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
- (PyDictObject *)BUILTINS(),
- name);
+ (PyDictObject *)BUILTINS(),
+ name);
if (res == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ NAME_ERROR_MSG, name);
}
if (true) goto error;
}
@@ -4048,18 +4069,17 @@
if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error;
if (res == NULL) {
_PyEval_FormatExcCheckArg(
- tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
if (true) goto error;
}
}
}
null = NULL;
}
- STACK_GROW(1);
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ stack_pointer[0] = res;
+ if (oparg & 1) stack_pointer[1] = null;
+ stack_pointer += 1 + ((oparg & 1));
DISPATCH();
}
@@ -4067,8 +4087,10 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
PyObject *res;
PyObject *null = NULL;
+ /* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
uint16_t version = read_u16(&this_instr[2].cache);
@@ -4096,10 +4118,9 @@
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
}
- STACK_GROW(1);
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ stack_pointer[0] = res;
+ if (oparg & 1) stack_pointer[1] = null;
+ stack_pointer += 1 + ((oparg & 1));
DISPATCH();
}
@@ -4107,8 +4128,10 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_MODULE);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
PyObject *res;
PyObject *null = NULL;
+ /* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
uint16_t version = read_u16(&this_instr[2].cache);
@@ -4117,6 +4140,7 @@
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
+ /* Skip 1 cache entry */
// _LOAD_GLOBAL_MODULE
{
uint16_t index = read_u16(&this_instr[4].cache);
@@ -4128,10 +4152,9 @@
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
}
- STACK_GROW(1);
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ stack_pointer[0] = res;
+ if (oparg & 1) stack_pointer[1] = null;
+ stack_pointer += 1 + ((oparg & 1));
DISPATCH();
}
@@ -4147,8 +4170,8 @@
if (true) goto error;
}
Py_INCREF(locals);
- STACK_GROW(1);
- stack_pointer[-1] = locals;
+ stack_pointer[0] = locals;
+ stack_pointer += 1;
DISPATCH();
}
@@ -4177,14 +4200,14 @@
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
- tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
GOTO_ERROR(error);
}
}
}
- STACK_GROW(1);
- stack_pointer[-1] = v;
+ stack_pointer[0] = v;
+ stack_pointer += 1;
DISPATCH();
}
@@ -4194,7 +4217,6 @@
INSTRUCTION_STATS(LOAD_SUPER_ATTR);
PREDICTED(LOAD_SUPER_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 2;
- static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
PyObject *class;
PyObject *global_super;
PyObject *self;
@@ -4224,8 +4246,8 @@
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
int err = _Py_call_instrumentation_2args(
- tstate, PY_MONITORING_EVENT_CALL,
- frame, this_instr, global_super, arg);
+ tstate, PY_MONITORING_EVENT_CALL,
+ frame, this_instr, global_super, arg);
if (err) goto pop_3_error;
}
// we make no attempt to optimize here; specializations should
@@ -4258,10 +4280,9 @@
if (attr == NULL) goto pop_3_error;
null = NULL;
}
- STACK_SHRINK(2);
- STACK_GROW(((oparg & 1) ? 1 : 0));
- stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
- if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; }
+ stack_pointer[-3] = attr;
+ if (oparg & 1) stack_pointer[-2] = null;
+ stack_pointer += -2 + ((oparg & 1));
DISPATCH();
}
@@ -4269,6 +4290,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
PyObject *self;
PyObject *class;
PyObject *global_super;
@@ -4286,8 +4308,8 @@
Py_DECREF(class);
Py_DECREF(self);
if (attr == NULL) goto pop_3_error;
- STACK_SHRINK(2);
- stack_pointer[-1] = attr;
+ stack_pointer[-3] = attr;
+ stack_pointer += -2 + (((0) ? 1 : 0));
DISPATCH();
}
@@ -4295,6 +4317,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD);
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
PyObject *self;
PyObject *class;
PyObject *global_super;
@@ -4324,9 +4347,9 @@
Py_DECREF(self);
self_or_null = NULL;
}
- STACK_SHRINK(1);
- stack_pointer[-2] = attr;
- stack_pointer[-1] = self_or_null;
+ stack_pointer[-3] = attr;
+ stack_pointer[-2] = self_or_null;
+ stack_pointer += -1;
DISPATCH();
}
@@ -4352,17 +4375,14 @@
PyObject *codeobj;
PyObject *func;
codeobj = stack_pointer[-1];
-
PyFunctionObject *func_obj = (PyFunctionObject *)
- PyFunction_New(codeobj, GLOBALS());
-
+ PyFunction_New(codeobj, GLOBALS());
Py_DECREF(codeobj);
if (func_obj == NULL) {
GOTO_ERROR(error);
}
-
_PyFunction_SetVersion(
- func_obj, ((PyCodeObject *)codeobj)->co_version);
+ func_obj, ((PyCodeObject *)codeobj)->co_version);
func = (PyObject *)func_obj;
stack_pointer[-1] = func;
DISPATCH();
@@ -4382,7 +4402,7 @@
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error;
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -4409,10 +4429,11 @@
}
else {
if (_PyErr_Occurred(tstate)) goto pop_3_error;
+ // Error!
attrs = Py_None; // Failure!
}
- STACK_SHRINK(2);
- stack_pointer[-1] = attrs;
+ stack_pointer[-3] = attrs;
+ stack_pointer += -2;
DISPATCH();
}
@@ -4428,8 +4449,8 @@
// On successful match, PUSH(values). Otherwise, PUSH(None).
values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
if (values_or_none == NULL) goto error;
- STACK_GROW(1);
- stack_pointer[-1] = values_or_none;
+ stack_pointer[0] = values_or_none;
+ stack_pointer += 1;
DISPATCH();
}
@@ -4442,8 +4463,8 @@
subject = stack_pointer[-1];
int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
res = match ? Py_True : Py_False;
- STACK_GROW(1);
- stack_pointer[-1] = res;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
DISPATCH();
}
@@ -4456,8 +4477,8 @@
subject = stack_pointer[-1];
int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
res = match ? Py_True : Py_False;
- STACK_GROW(1);
- stack_pointer[-1] = res;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
DISPATCH();
}
@@ -4476,7 +4497,7 @@
exc_value = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
Py_XSETREF(exc_info->exc_value, exc_value);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -4492,7 +4513,7 @@
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -4524,7 +4545,7 @@
#endif
JUMPBY(oparg * flag);
}
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -4556,7 +4577,7 @@
#endif
JUMPBY(oparg * flag);
}
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -4572,7 +4593,7 @@
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -4583,7 +4604,7 @@
PyObject *value;
value = stack_pointer[-1];
Py_DECREF(value);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -4603,9 +4624,9 @@
}
assert(PyExceptionInstance_Check(new_exc));
exc_info->exc_value = Py_NewRef(new_exc);
- STACK_GROW(1);
- stack_pointer[-2] = prev_exc;
- stack_pointer[-1] = new_exc;
+ stack_pointer[-1] = prev_exc;
+ stack_pointer[0] = new_exc;
+ stack_pointer += 1;
DISPATCH();
}
@@ -4615,8 +4636,8 @@
INSTRUCTION_STATS(PUSH_NULL);
PyObject *res;
res = NULL;
- STACK_GROW(1);
- stack_pointer[-1] = res;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
DISPATCH();
}
@@ -4625,29 +4646,29 @@
next_instr += 1;
INSTRUCTION_STATS(RAISE_VARARGS);
PyObject **args;
- args = stack_pointer - oparg;
+ args = &stack_pointer[-oparg];
TIER_ONE_ONLY
PyObject *cause = NULL, *exc = NULL;
switch (oparg) {
- case 2:
+ case 2:
cause = args[1];
/* fall through */
- case 1:
+ case 1:
exc = args[0];
/* fall through */
- case 0:
+ case 0:
if (do_raise(tstate, exc, cause)) {
assert(oparg == 0);
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
break;
- default:
+ default:
_PyErr_SetString(tstate, PyExc_SystemError,
"bad RAISE_VARARGS oparg");
break;
}
- if (true) { STACK_SHRINK(oparg); goto error; }
+ if (true) { stack_pointer += -oparg; goto error; }
}
TARGET(RERAISE) {
@@ -4657,7 +4678,7 @@
PyObject *exc;
PyObject **values;
exc = stack_pointer[-1];
- values = stack_pointer - 1 - oparg;
+ values = &stack_pointer[-1 - oparg];
TIER_ONE_ONLY
assert(oparg >= 0 && oparg <= 2);
if (oparg) {
@@ -4693,12 +4714,11 @@
INSTRUCTION_STATS(RESUME);
PREDICTED(RESUME);
_Py_CODEUNIT *this_instr = next_instr - 1;
- static_assert(0 == 0, "incorrect cache size");
TIER_ONE_ONLY
assert(frame == tstate->current_frame);
uintptr_t global_version =
- _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) &
- ~_PY_EVAL_EVENTS_MASK;
+ _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) &
+ ~_PY_EVAL_EVENTS_MASK;
uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
assert((code_version & 255) == 0);
if (code_version != global_version) {
@@ -4719,10 +4739,11 @@
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME_CHECK);
-#if defined(__EMSCRIPTEN__)
+ static_assert(0 == 0, "incorrect cache size");
+ #if defined(__EMSCRIPTEN__)
DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME);
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
-#endif
+ #endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker);
uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
@@ -4736,7 +4757,7 @@
INSTRUCTION_STATS(RETURN_CONST);
PyObject *value;
PyObject *retval;
- // LOAD_CONST
+ // _LOAD_CONST
{
value = GETITEM(FRAME_CO_CONSTS, oparg);
Py_INCREF(value);
@@ -4744,11 +4765,11 @@
// _POP_FRAME
retval = value;
{
- assert(EMPTY());
#if TIER_ONE
assert(frame != &entry_frame);
#endif
- STORE_SP();
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
@@ -4757,12 +4778,12 @@
_PyFrame_StackPush(frame, retval);
LOAD_SP();
LOAD_IP(frame->return_offset);
- #if LLTRACE && TIER_ONE
+ #if LLTRACE && TIER_ONE
lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
if (lltrace < 0) {
goto exit_unwind;
}
- #endif
+ #endif
}
DISPATCH();
}
@@ -4801,12 +4822,12 @@
INSTRUCTION_STATS(RETURN_VALUE);
PyObject *retval;
retval = stack_pointer[-1];
- STACK_SHRINK(1);
- assert(EMPTY());
#if TIER_ONE
assert(frame != &entry_frame);
#endif
- STORE_SP();
+ stack_pointer += -1;
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
@@ -4815,12 +4836,12 @@
_PyFrame_StackPush(frame, retval);
LOAD_SP();
LOAD_IP(frame->return_offset);
-#if LLTRACE && TIER_ONE
+ #if LLTRACE && TIER_ONE
lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
if (lltrace < 0) {
goto exit_unwind;
}
-#endif
+ #endif
DISPATCH();
}
@@ -4830,7 +4851,6 @@
INSTRUCTION_STATS(SEND);
PREDICTED(SEND);
_Py_CODEUNIT *this_instr = next_instr - 2;
- static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
PyObject *receiver;
PyObject *v;
PyObject *retval;
@@ -4897,6 +4917,7 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND_GEN);
+ static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
PyObject *v;
PyObject *receiver;
v = stack_pointer[-1];
@@ -4955,7 +4976,7 @@
int err = PySet_Add(set, v);
Py_DECREF(v);
if (err) goto pop_1_error;
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -4971,28 +4992,28 @@
PyFunctionObject *func_obj = (PyFunctionObject *)func;
switch(oparg) {
case MAKE_FUNCTION_CLOSURE:
- assert(func_obj->func_closure == NULL);
- func_obj->func_closure = attr;
- break;
+ assert(func_obj->func_closure == NULL);
+ func_obj->func_closure = attr;
+ break;
case MAKE_FUNCTION_ANNOTATIONS:
- assert(func_obj->func_annotations == NULL);
- func_obj->func_annotations = attr;
- break;
+ assert(func_obj->func_annotations == NULL);
+ func_obj->func_annotations = attr;
+ break;
case MAKE_FUNCTION_KWDEFAULTS:
- assert(PyDict_CheckExact(attr));
- assert(func_obj->func_kwdefaults == NULL);
- func_obj->func_kwdefaults = attr;
- break;
+ assert(PyDict_CheckExact(attr));
+ assert(func_obj->func_kwdefaults == NULL);
+ func_obj->func_kwdefaults = attr;
+ break;
case MAKE_FUNCTION_DEFAULTS:
- assert(PyTuple_CheckExact(attr));
- assert(func_obj->func_defaults == NULL);
- func_obj->func_defaults = attr;
- break;
+ assert(PyTuple_CheckExact(attr));
+ assert(func_obj->func_defaults == NULL);
+ func_obj->func_defaults = attr;
+ break;
default:
- Py_UNREACHABLE();
+ Py_UNREACHABLE();
}
- STACK_SHRINK(1);
- stack_pointer[-1] = func;
+ stack_pointer[-2] = func;
+ stack_pointer += -1;
DISPATCH();
}
@@ -5007,7 +5028,7 @@
int err = _PySet_Update(set, iterable);
Py_DECREF(iterable);
if (err < 0) goto pop_1_error;
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -5017,7 +5038,6 @@
INSTRUCTION_STATS(STORE_ATTR);
PREDICTED(STORE_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 5;
- static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
PyObject *owner;
PyObject *v;
// _SPECIALIZE_STORE_ATTR
@@ -5045,7 +5065,7 @@
Py_DECREF(owner);
if (err) goto pop_2_error;
}
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -5053,8 +5073,10 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE);
+ static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
PyObject *owner;
PyObject *value;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -5086,7 +5108,7 @@
}
Py_DECREF(owner);
}
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -5094,8 +5116,10 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_SLOT);
+ static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
PyObject *owner;
PyObject *value;
+ /* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
@@ -5115,7 +5139,7 @@
Py_XDECREF(old_value);
Py_DECREF(owner);
}
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -5123,6 +5147,7 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_WITH_HINT);
+ static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
PyObject *owner;
PyObject *value;
owner = stack_pointer[-1];
@@ -5167,7 +5192,7 @@
/* PEP 509 */
dict->ma_version_tag = new_version;
Py_DECREF(owner);
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -5181,7 +5206,7 @@
PyObject *oldobj = PyCell_GET(cell);
PyCell_SET(cell, v);
Py_XDECREF(oldobj);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -5192,7 +5217,7 @@
PyObject *value;
value = stack_pointer[-1];
SETLOCAL(oparg, value);
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -5224,7 +5249,7 @@
uint32_t oparg2 = oparg & 15;
SETLOCAL(oparg1, value1);
SETLOCAL(oparg2, value2);
- STACK_SHRINK(2);
+ stack_pointer += -2;
DISPATCH();
}
@@ -5238,7 +5263,7 @@
int err = PyDict_SetItem(GLOBALS(), name, v);
Py_DECREF(v);
if (err) goto pop_1_error;
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -5258,12 +5283,12 @@
if (true) goto pop_1_error;
}
if (PyDict_CheckExact(ns))
- err = PyDict_SetItem(ns, name, v);
+ err = PyDict_SetItem(ns, name, v);
else
- err = PyObject_SetItem(ns, name, v);
+ err = PyObject_SetItem(ns, name, v);
Py_DECREF(v);
if (err) goto pop_1_error;
- STACK_SHRINK(1);
+ stack_pointer += -1;
DISPATCH();
}
@@ -5291,7 +5316,7 @@
Py_DECREF(v);
Py_DECREF(container);
if (err) goto pop_4_error;
- STACK_SHRINK(4);
+ stack_pointer += -4;
DISPATCH();
}
@@ -5301,7 +5326,6 @@
INSTRUCTION_STATS(STORE_SUBSCR);
PREDICTED(STORE_SUBSCR);
_Py_CODEUNIT *this_instr = next_instr - 2;
- static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *container;
PyObject *v;
@@ -5331,7 +5355,7 @@
Py_DECREF(sub);
if (err) goto pop_3_error;
}
- STACK_SHRINK(3);
+ stack_pointer += -3;
DISPATCH();
}
@@ -5339,6 +5363,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_DICT);
+ static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *dict;
PyObject *value;
@@ -5350,7 +5375,7 @@
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
if (err) goto pop_3_error;
- STACK_SHRINK(3);
+ stack_pointer += -3;
DISPATCH();
}
@@ -5358,6 +5383,7 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT);
+ static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *list;
PyObject *value;
@@ -5366,21 +5392,19 @@
value = stack_pointer[-3];
DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
-
// Ensure nonnegative, zero-or-one-digit ints.
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
// Ensure index < len(list)
DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
STAT_INC(STORE_SUBSCR, hit);
-
PyObject *old_value = PyList_GET_ITEM(list, index);
PyList_SET_ITEM(list, index, value);
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
- STACK_SHRINK(3);
+ stack_pointer += -3;
DISPATCH();
}
@@ -5404,7 +5428,6 @@
INSTRUCTION_STATS(TO_BOOL);
PREDICTED(TO_BOOL);
_Py_CODEUNIT *this_instr = next_instr - 4;
- static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
// _SPECIALIZE_TO_BOOL
@@ -5437,6 +5460,7 @@
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE);
+ static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
@@ -5455,6 +5479,7 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_BOOL);
+ static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
value = stack_pointer[-1];
DEOPT_IF(!PyBool_Check(value), TO_BOOL);
@@ -5466,6 +5491,7 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_INT);
+ static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
@@ -5487,6 +5513,7 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_LIST);
+ static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
@@ -5502,6 +5529,7 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_NONE);
+ static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
@@ -5517,6 +5545,7 @@
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_STR);
+ static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
@@ -5587,7 +5616,7 @@
int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
Py_DECREF(seq);
if (res == 0) goto pop_1_error;
- STACK_GROW((oparg & 0xFF) + (oparg >> 8));
+ stack_pointer += (oparg >> 8) + (oparg & 0xFF);
DISPATCH();
}
@@ -5597,7 +5626,6 @@
INSTRUCTION_STATS(UNPACK_SEQUENCE);
PREDICTED(UNPACK_SEQUENCE);
_Py_CODEUNIT *this_instr = next_instr - 2;
- static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
PyObject *seq;
// _SPECIALIZE_UNPACK_SEQUENCE
seq = stack_pointer[-1];
@@ -5623,8 +5651,7 @@
Py_DECREF(seq);
if (res == 0) goto pop_1_error;
}
- STACK_SHRINK(1);
- STACK_GROW(oparg);
+ stack_pointer += -1 + oparg;
DISPATCH();
}
@@ -5632,10 +5659,11 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST);
+ static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
PyObject *seq;
PyObject **values;
seq = stack_pointer[-1];
- values = stack_pointer - 1;
+ values = &stack_pointer[-1];
DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
@@ -5644,8 +5672,7 @@
*values++ = Py_NewRef(items[i]);
}
Py_DECREF(seq);
- STACK_SHRINK(1);
- STACK_GROW(oparg);
+ stack_pointer += -1 + oparg;
DISPATCH();
}
@@ -5653,10 +5680,11 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE);
+ static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
PyObject *seq;
PyObject **values;
seq = stack_pointer[-1];
- values = stack_pointer - 1;
+ values = &stack_pointer[-1];
DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
@@ -5665,8 +5693,7 @@
*values++ = Py_NewRef(items[i]);
}
Py_DECREF(seq);
- STACK_SHRINK(1);
- STACK_GROW(oparg);
+ stack_pointer += -1 + oparg;
DISPATCH();
}
@@ -5674,10 +5701,11 @@
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE);
+ static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
PyObject *seq;
PyObject **values;
seq = stack_pointer[-1];
- values = stack_pointer - 1;
+ values = &stack_pointer[-1];
DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE);
assert(oparg == 2);
@@ -5685,8 +5713,7 @@
values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
Py_DECREF(seq);
- STACK_SHRINK(1);
- STACK_GROW(oparg);
+ stack_pointer += -1 + oparg;
DISPATCH();
}
@@ -5708,9 +5735,8 @@
- exit_func: FOURTH = the context.__exit__ bound method
We call FOURTH(type(TOP), TOP, GetTraceback(TOP)).
Then we push the __exit__ return value.
- */
+ */
PyObject *exc, *tb;
-
assert(val && PyExceptionInstance_Check(val));
exc = PyExceptionInstance_Class(val);
tb = PyException_GetTraceback(val);
@@ -5724,10 +5750,10 @@
(void)lasti; // Shut up compiler warning if asserts are off
PyObject *stack[4] = {NULL, exc, val, tb};
res = PyObject_Vectorcall(exit_func, stack + 1,
- 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
+ 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
if (res == NULL) goto error;
- STACK_GROW(1);
- stack_pointer[-1] = res;
+ stack_pointer[0] = res;
+ stack_pointer += 1;
DISPATCH();
}
@@ -5759,5 +5785,4 @@
LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND);
goto resume_frame;
}
-
#undef TIER_ONE
diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py
new file mode 100644
index 0000000..027f986
--- /dev/null
+++ b/Tools/cases_generator/analyzer.py
@@ -0,0 +1,456 @@
+from dataclasses import dataclass
+import lexer
+import parser
+from typing import Optional
+
+
+@dataclass
+class Properties:
+ escapes: bool
+ infallible: bool
+ deopts: bool
+ oparg: bool
+ jumps: bool
+ ends_with_eval_breaker: bool
+ needs_this: bool
+ always_exits: bool
+ stores_sp: bool
+
+ def dump(self, indent: str) -> None:
+ print(indent, end="")
+ text = ", ".join([f"{key}: {value}" for (key, value) in self.__dict__.items()])
+ print(indent, text, sep="")
+
+ @staticmethod
+ def from_list(properties: list["Properties"]) -> "Properties":
+ return Properties(
+ escapes=any(p.escapes for p in properties),
+ infallible=all(p.infallible for p in properties),
+ deopts=any(p.deopts for p in properties),
+ oparg=any(p.oparg for p in properties),
+ jumps=any(p.jumps for p in properties),
+ ends_with_eval_breaker=any(p.ends_with_eval_breaker for p in properties),
+ needs_this=any(p.needs_this for p in properties),
+ always_exits=any(p.always_exits for p in properties),
+ stores_sp=any(p.stores_sp for p in properties),
+ )
+
+
+SKIP_PROPERTIES = Properties(
+ escapes=False,
+ infallible=True,
+ deopts=False,
+ oparg=False,
+ jumps=False,
+ ends_with_eval_breaker=False,
+ needs_this=False,
+ always_exits=False,
+ stores_sp=False,
+)
+
+
+@dataclass
+class Skip:
+ "Unused cache entry"
+ size: int
+
+ @property
+ def name(self) -> str:
+ return f"unused/{self.size}"
+
+ @property
+ def properties(self) -> Properties:
+ return SKIP_PROPERTIES
+
+
+@dataclass
+class StackItem:
+ name: str
+ type: str | None
+ condition: str | None
+ size: str
+ peek: bool = False
+
+ def __str__(self) -> str:
+ cond = f" if ({self.condition})" if self.condition else ""
+ size = f"[{self.size}]" if self.size != "1" else ""
+ type = "" if self.type is None else f"{self.type} "
+ return f"{type}{self.name}{size}{cond} {self.peek}"
+
+ def is_array(self) -> bool:
+ return self.type == "PyObject **"
+
+
+@dataclass
+class StackEffect:
+ inputs: list[StackItem]
+ outputs: list[StackItem]
+
+ def __str__(self) -> str:
+ return f"({', '.join([str(i) for i in self.inputs])} -- {', '.join([str(i) for i in self.outputs])})"
+
+
+@dataclass
+class CacheEntry:
+ name: str
+ size: int
+
+ def __str__(self) -> str:
+ return f"{self.name}/{self.size}"
+
+
+@dataclass
+class Uop:
+ name: str
+ context: parser.Context | None
+ annotations: list[str]
+ stack: StackEffect
+ caches: list[CacheEntry]
+ body: list[lexer.Token]
+ properties: Properties
+ _size: int = -1
+
+ def dump(self, indent: str) -> None:
+ print(
+ indent, self.name, ", ".join(self.annotations) if self.annotations else ""
+ )
+ print(indent, self.stack, ", ".join([str(c) for c in self.caches]))
+ self.properties.dump(" " + indent)
+
+ @property
+ def size(self) -> int:
+ if self._size < 0:
+ self._size = sum(c.size for c in self.caches)
+ return self._size
+
+
+Part = Uop | Skip
+
+
+@dataclass
+class Instruction:
+ name: str
+ parts: list[Part]
+ _properties: Properties | None
+ is_target: bool = False
+ family: Optional["Family"] = None
+
+ @property
+ def properties(self) -> Properties:
+ if self._properties is None:
+ self._properties = self._compute_properties()
+ return self._properties
+
+ def _compute_properties(self) -> Properties:
+ return Properties.from_list([part.properties for part in self.parts])
+
+ def dump(self, indent: str) -> None:
+ print(indent, self.name, "=", ", ".join([part.name for part in self.parts]))
+ self.properties.dump(" " + indent)
+
+ @property
+ def size(self) -> int:
+ return 1 + sum(part.size for part in self.parts)
+
+
+@dataclass
+class PseudoInstruction:
+ name: str
+ targets: list[Instruction]
+ flags: list[str]
+
+ def dump(self, indent: str) -> None:
+ print(indent, self.name, "->", " or ".join([t.name for t in self.targets]))
+
+
+@dataclass
+class Family:
+ name: str
+ size: str
+ members: list[Instruction]
+
+ def dump(self, indent: str) -> None:
+ print(indent, self.name, "= ", ", ".join([m.name for m in self.members]))
+
+
+@dataclass
+class Analysis:
+ instructions: dict[str, Instruction]
+ uops: dict[str, Uop]
+ families: dict[str, Family]
+ pseudos: dict[str, PseudoInstruction]
+
+
+def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError:
+ # To do -- support file and line output
+ # Construct a SyntaxError instance from message and token
+ return lexer.make_syntax_error(message, "", tkn.line, tkn.column, "")
+
+
+def override_error(
+ name: str,
+ context: parser.Context | None,
+ prev_context: parser.Context | None,
+ token: lexer.Token,
+) -> SyntaxError:
+ return analysis_error(
+ f"Duplicate definition of '{name}' @ {context} "
+ f"previous definition @ {prev_context}",
+ token,
+ )
+
+
+def convert_stack_item(item: parser.StackEffect) -> StackItem:
+ return StackItem(item.name, item.type, item.cond, (item.size or "1"))
+
+
+def analyze_stack(op: parser.InstDef) -> StackEffect:
+ inputs: list[StackItem] = [
+ convert_stack_item(i) for i in op.inputs if isinstance(i, parser.StackEffect)
+ ]
+ outputs: list[StackItem] = [convert_stack_item(i) for i in op.outputs]
+ for input, output in zip(inputs, outputs):
+ if input.name == output.name:
+ input.peek = output.peek = True
+ return StackEffect(inputs, outputs)
+
+
+def analyze_caches(op: parser.InstDef) -> list[CacheEntry]:
+ caches: list[parser.CacheEffect] = [
+ i for i in op.inputs if isinstance(i, parser.CacheEffect)
+ ]
+ return [CacheEntry(i.name, int(i.size)) for i in caches]
+
+
+def variable_used(node: parser.InstDef, name: str) -> bool:
+ """Determine whether a variable with a given name is used in a node."""
+ return any(
+ token.kind == "IDENTIFIER" and token.text == name for token in node.tokens
+ )
+
+
+def is_infallible(op: parser.InstDef) -> bool:
+ return not (
+ variable_used(op, "ERROR_IF")
+ or variable_used(op, "error")
+ or variable_used(op, "pop_1_error")
+ or variable_used(op, "exception_unwind")
+ or variable_used(op, "resume_with_error")
+ )
+
+
+from flags import makes_escaping_api_call
+
+EXITS = {
+ "DISPATCH",
+ "GO_TO_INSTRUCTION",
+ "Py_UNREACHABLE",
+ "DISPATCH_INLINED",
+ "DISPATCH_GOTO",
+}
+
+
+def eval_breaker_at_end(op: parser.InstDef) -> bool:
+ return op.tokens[-5].text == "CHECK_EVAL_BREAKER"
+
+
+def always_exits(op: parser.InstDef) -> bool:
+ depth = 0
+ tkn_iter = iter(op.tokens)
+ for tkn in tkn_iter:
+ if tkn.kind == "LBRACE":
+ depth += 1
+ elif tkn.kind == "RBRACE":
+ depth -= 1
+ elif depth > 1:
+ continue
+ elif tkn.kind == "GOTO" or tkn.kind == "RETURN":
+ return True
+ elif tkn.kind == "KEYWORD":
+ if tkn.text in EXITS:
+ return True
+ elif tkn.kind == "IDENTIFIER":
+ if tkn.text in EXITS:
+ return True
+ if tkn.text == "DEOPT_IF" or tkn.text == "ERROR_IF":
+ next(tkn_iter) # '('
+ t = next(tkn_iter)
+ if t.text == "true":
+ return True
+ return False
+
+
+def compute_properties(op: parser.InstDef) -> Properties:
+ return Properties(
+ escapes=makes_escaping_api_call(op),
+ infallible=is_infallible(op),
+ deopts=variable_used(op, "DEOPT_IF"),
+ oparg=variable_used(op, "oparg"),
+ jumps=variable_used(op, "JUMPBY"),
+ ends_with_eval_breaker=eval_breaker_at_end(op),
+ needs_this=variable_used(op, "this_instr"),
+ always_exits=always_exits(op),
+ stores_sp=variable_used(op, "STORE_SP"),
+ )
+
+
+def make_uop(name: str, op: parser.InstDef) -> Uop:
+ return Uop(
+ name=name,
+ context=op.context,
+ annotations=op.annotations,
+ stack=analyze_stack(op),
+ caches=analyze_caches(op),
+ body=op.block.tokens,
+ properties=compute_properties(op),
+ )
+
+
+def add_op(op: parser.InstDef, uops: dict[str, Uop]) -> None:
+ assert op.kind == "op"
+ if op.name in uops:
+ if "override" not in op.annotations:
+ raise override_error(
+ op.name, op.context, uops[op.name].context, op.tokens[0]
+ )
+ uops[op.name] = make_uop(op.name, op)
+
+
+def add_instruction(
+ name: str, parts: list[Part], instructions: dict[str, Instruction]
+) -> None:
+ instructions[name] = Instruction(name, parts, None)
+
+
+def desugar_inst(
+ inst: parser.InstDef, instructions: dict[str, Instruction], uops: dict[str, Uop]
+) -> None:
+ assert inst.kind == "inst"
+ name = inst.name
+ uop = make_uop("_" + inst.name, inst)
+ uops[inst.name] = uop
+ add_instruction(name, [uop], instructions)
+
+
+def add_macro(
+ macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop]
+) -> None:
+ parts: list[Uop | Skip] = []
+ for part in macro.uops:
+ match part:
+ case parser.OpName():
+ if part.name not in uops:
+ analysis_error(f"No Uop named {part.name}", macro.tokens[0])
+ parts.append(uops[part.name])
+ case parser.CacheEffect():
+ parts.append(Skip(part.size))
+ case _:
+ assert False
+ assert parts
+ add_instruction(macro.name, parts, instructions)
+
+
+def add_family(
+ pfamily: parser.Family,
+ instructions: dict[str, Instruction],
+ families: dict[str, Family],
+) -> None:
+ family = Family(
+ pfamily.name,
+ pfamily.size,
+ [instructions[member_name] for member_name in pfamily.members],
+ )
+ for member in family.members:
+ member.family = family
+ # The head of the family is an implicit jump target for DEOPTs
+ instructions[family.name].is_target = True
+ families[family.name] = family
+
+
+def add_pseudo(
+ pseudo: parser.Pseudo,
+ instructions: dict[str, Instruction],
+ pseudos: dict[str, PseudoInstruction],
+) -> None:
+ pseudos[pseudo.name] = PseudoInstruction(
+ pseudo.name,
+ [instructions[target] for target in pseudo.targets],
+ pseudo.flags,
+ )
+
+
+def analyze_forest(forest: list[parser.AstNode]) -> Analysis:
+ instructions: dict[str, Instruction] = {}
+ uops: dict[str, Uop] = {}
+ families: dict[str, Family] = {}
+ pseudos: dict[str, PseudoInstruction] = {}
+ for node in forest:
+ match node:
+ case parser.InstDef(name):
+ if node.kind == "inst":
+ desugar_inst(node, instructions, uops)
+ else:
+ assert node.kind == "op"
+ add_op(node, uops)
+ case parser.Macro():
+ pass
+ case parser.Family():
+ pass
+ case parser.Pseudo():
+ pass
+ case _:
+ assert False
+ for node in forest:
+ if isinstance(node, parser.Macro):
+ add_macro(node, instructions, uops)
+ for node in forest:
+ match node:
+ case parser.Family():
+ add_family(node, instructions, families)
+ case parser.Pseudo():
+ add_pseudo(node, instructions, pseudos)
+ case _:
+ pass
+ for uop in uops.values():
+ tkn_iter = iter(uop.body)
+ for tkn in tkn_iter:
+ if tkn.kind == "IDENTIFIER" and tkn.text == "GO_TO_INSTRUCTION":
+ if next(tkn_iter).kind != "LPAREN":
+ continue
+ target = next(tkn_iter)
+ if target.kind != "IDENTIFIER":
+ continue
+ if target.text in instructions:
+ instructions[target.text].is_target = True
+ # Hack
+ instructions["BINARY_OP_INPLACE_ADD_UNICODE"].family = families["BINARY_OP"]
+ return Analysis(instructions, uops, families, pseudos)
+
+
+def analyze_files(filenames: list[str]) -> Analysis:
+ return analyze_forest(parser.parse_files(filenames))
+
+
+def dump_analysis(analysis: Analysis) -> None:
+ print("Uops:")
+ for u in analysis.uops.values():
+ u.dump(" ")
+ print("Instructions:")
+ for i in analysis.instructions.values():
+ i.dump(" ")
+ print("Families:")
+ for f in analysis.families.values():
+ f.dump(" ")
+ print("Pseudos:")
+ for p in analysis.pseudos.values():
+ p.dump(" ")
+
+
+if __name__ == "__main__":
+ import sys
+
+ if len(sys.argv) < 2:
+ print("No input")
+ else:
+ filenames = sys.argv[1:]
+ dump_analysis(analyze_files(filenames))
diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py
new file mode 100644
index 0000000..0b7edd0
--- /dev/null
+++ b/Tools/cases_generator/cwriter.py
@@ -0,0 +1,111 @@
+from lexer import Token
+from typing import TextIO
+
+
+class CWriter:
+ "A writer that understands tokens and how to format C code"
+
+ last_token: Token | None
+
+ def __init__(self, out: TextIO, indent: int, line_directives: bool):
+ self.out = out
+ self.base_column = indent * 4
+ self.indents = [i * 4 for i in range(indent + 1)]
+ self.line_directives = line_directives
+ self.last_token = None
+ self.newline = True
+
+ def set_position(self, tkn: Token) -> None:
+ if self.last_token is not None:
+ if self.last_token.line < tkn.line:
+ self.out.write("\n")
+ if self.line_directives:
+ self.out.write(f'#line {tkn.line} "{tkn.filename}"\n')
+ self.out.write(" " * self.indents[-1])
+ else:
+ gap = tkn.column - self.last_token.end_column
+ self.out.write(" " * gap)
+ elif self.newline:
+ self.out.write(" " * self.indents[-1])
+ self.last_token = tkn
+ self.newline = False
+
+ def emit_at(self, txt: str, where: Token) -> None:
+ self.set_position(where)
+ self.out.write(txt)
+
+ def maybe_dedent(self, txt: str) -> None:
+ parens = txt.count("(") - txt.count(")")
+ if parens < 0:
+ self.indents.pop()
+ elif "}" in txt or is_label(txt):
+ self.indents.pop()
+
+ def maybe_indent(self, txt: str) -> None:
+ parens = txt.count("(") - txt.count(")")
+ if parens > 0 and self.last_token:
+ offset = self.last_token.end_column - 1
+ if offset <= self.indents[-1] or offset > 40:
+ offset = self.indents[-1] + 4
+ self.indents.append(offset)
+ elif "{" in txt or is_label(txt):
+ self.indents.append(self.indents[-1] + 4)
+
+ def emit_text(self, txt: str) -> None:
+ self.out.write(txt)
+
+ def emit_multiline_comment(self, tkn: Token) -> None:
+ self.set_position(tkn)
+ lines = tkn.text.splitlines(True)
+ first = True
+ for line in lines:
+ text = line.lstrip()
+ if first:
+ spaces = 0
+ else:
+ spaces = self.indents[-1]
+ if text.startswith("*"):
+ spaces += 1
+ else:
+ spaces += 3
+ first = False
+ self.out.write(" " * spaces)
+ self.out.write(text)
+
+ def emit_token(self, tkn: Token) -> None:
+ if tkn.kind == "COMMENT" and "\n" in tkn.text:
+ return self.emit_multiline_comment(tkn)
+ self.maybe_dedent(tkn.text)
+ self.set_position(tkn)
+ self.emit_text(tkn.text)
+ self.maybe_indent(tkn.text)
+
+ def emit_str(self, txt: str) -> None:
+ self.maybe_dedent(txt)
+ if self.newline and txt:
+ if txt[0] != "\n":
+ self.out.write(" " * self.indents[-1])
+ self.newline = False
+ self.emit_text(txt)
+ if txt.endswith("\n"):
+ self.newline = True
+ self.maybe_indent(txt)
+ self.last_token = None
+
+ def emit(self, txt: str | Token) -> None:
+ if isinstance(txt, Token):
+ self.emit_token(txt)
+ elif isinstance(txt, str):
+ self.emit_str(txt)
+ else:
+ assert False
+
+ def start_line(self) -> None:
+ if not self.newline:
+ self.out.write("\n")
+ self.newline = True
+ self.last_token = None
+
+
+def is_label(txt: str) -> bool:
+ return not txt.startswith("//") and txt.endswith(":")
diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py
index 1f94c1f..4b7f028 100644
--- a/Tools/cases_generator/generate_cases.py
+++ b/Tools/cases_generator/generate_cases.py
@@ -883,7 +883,6 @@ def main() -> None:
return
# These raise OSError if output can't be written
- a.write_instructions(args.output, args.emit_line_directives)
a.assign_opcode_ids()
a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h)
diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py
index 1185c85..c3c2954 100644
--- a/Tools/cases_generator/lexer.py
+++ b/Tools/cases_generator/lexer.py
@@ -112,7 +112,7 @@ STRING = "STRING"
char = r"\'.\'" # TODO: escape sequence
CHARACTER = "CHARACTER"
-comment_re = r"//.*|/\*([^*]|\*[^/])*\*/"
+comment_re = r"(//.*)|/\*([^*]|\*[^/])*\*/"
COMMENT = "COMMENT"
newline = r"\n"
@@ -234,6 +234,7 @@ def make_syntax_error(
@dataclass(slots=True)
class Token:
+ filename: str
kind: str
text: str
begin: tuple[int, int]
@@ -261,7 +262,7 @@ class Token:
def replaceText(self, txt: str) -> "Token":
assert isinstance(txt, str)
- return Token(self.kind, txt, self.begin, self.end)
+ return Token(self.filename, self.kind, txt, self.begin, self.end)
def __repr__(self) -> str:
b0, b1 = self.begin
@@ -272,7 +273,7 @@ class Token:
return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})"
-def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]:
+def tokenize(src: str, line: int = 1, filename: str = "") -> Iterator[Token]:
linestart = -1
for m in matcher.finditer(src):
start, end = m.span()
@@ -323,7 +324,7 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T
else:
begin = line, start - linestart
if kind != "\n":
- yield Token(kind, text, begin, (line, start - linestart + len(text)))
+ yield Token(filename, kind, text, begin, (line, start - linestart + len(text)))
def to_text(tkns: list[Token], dedent: int = 0) -> str:
diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini
index e7175e2..8e5a318 100644
--- a/Tools/cases_generator/mypy.ini
+++ b/Tools/cases_generator/mypy.ini
@@ -11,3 +11,5 @@ strict = True
strict_concatenate = True
enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined
warn_unreachable = True
+allow_redefinition = True
+implicit_reexport = True
diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py
new file mode 100644
index 0000000..12173a6
--- /dev/null
+++ b/Tools/cases_generator/parser.py
@@ -0,0 +1,55 @@
+from parsing import (
+ InstDef,
+ Macro,
+ Pseudo,
+ Family,
+ Parser,
+ Context,
+ CacheEffect,
+ StackEffect,
+ OpName,
+ AstNode,
+)
+from formatting import prettify_filename
+
+
+BEGIN_MARKER = "// BEGIN BYTECODES //"
+END_MARKER = "// END BYTECODES //"
+
+
+def parse_files(filenames: list[str]) -> list[AstNode]:
+ result: list[AstNode] = []
+ for filename in filenames:
+ with open(filename) as file:
+ src = file.read()
+
+ psr = Parser(src, filename=prettify_filename(filename))
+
+ # Skip until begin marker
+ while tkn := psr.next(raw=True):
+ if tkn.text == BEGIN_MARKER:
+ break
+ else:
+ raise psr.make_syntax_error(
+ f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}"
+ )
+ start = psr.getpos()
+
+ # Find end marker, then delete everything after it
+ while tkn := psr.next(raw=True):
+ if tkn.text == END_MARKER:
+ break
+ del psr.tokens[psr.getpos() - 1 :]
+
+ # Parse from start
+ psr.setpos(start)
+ thing_first_token = psr.peek()
+ while node := psr.definition():
+ assert node is not None
+ result.append(node) # type: ignore[arg-type]
+ if not psr.eof():
+ psr.backup()
+ raise psr.make_syntax_error(
+ f"Extra stuff at the end of {filename}", psr.next(True)
+ )
+ return result
diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py
index 7800adf..60c185d 100644
--- a/Tools/cases_generator/parsing.py
+++ b/Tools/cases_generator/parsing.py
@@ -141,10 +141,11 @@ class Pseudo(Node):
flags: list[str] # instr flags to set on the pseudo instruction
targets: list[str] # opcodes this can be replaced by
+AstNode = InstDef | Macro | Pseudo | Family
class Parser(PLexer):
@contextual
- def definition(self) -> InstDef | Macro | Pseudo | Family | None:
+ def definition(self) -> AstNode | None:
if macro := self.macro_def():
return macro
if family := self.family_def():
diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py
new file mode 100644
index 0000000..9cd8e1a
--- /dev/null
+++ b/Tools/cases_generator/stack.py
@@ -0,0 +1,81 @@
+import sys
+from analyzer import StackItem
+from dataclasses import dataclass
+from formatting import maybe_parenthesize
+
+
+def var_size(var: StackItem) -> str:
+ if var.condition:
+ # Special case simplification
+ if var.condition == "oparg & 1" and var.size == "1":
+ return f"({var.condition})"
+ else:
+ return f"(({var.condition}) ? {var.size} : 0)"
+ else:
+ return var.size
+
+
+class StackOffset:
+ "The stack offset of the virtual base of the stack from the physical stack pointer"
+
+ def __init__(self) -> None:
+ self.popped: list[str] = []
+ self.pushed: list[str] = []
+
+ def pop(self, item: StackItem) -> None:
+ self.popped.append(var_size(item))
+
+ def push(self, item: StackItem) -> None:
+ self.pushed.append(var_size(item))
+
+ def simplify(self) -> None:
+ "Remove matching values from both the popped and pushed list"
+ if not self.popped or not self.pushed:
+ return
+ # Sort the list so the lexically largest element is last.
+ popped = sorted(self.popped)
+ pushed = sorted(self.pushed)
+ self.popped = []
+ self.pushed = []
+ while popped and pushed:
+ pop = popped.pop()
+ push = pushed.pop()
+ if pop == push:
+ pass
+ elif pop > push:
+ # if pop > push, there can be no element in pushed matching pop.
+ self.popped.append(pop)
+ pushed.append(push)
+ else:
+ self.pushed.append(push)
+ popped.append(pop)
+ self.popped.extend(popped)
+ self.pushed.extend(pushed)
+
+ def to_c(self) -> str:
+ self.simplify()
+ int_offset = 0
+ symbol_offset = ""
+ for item in self.popped:
+ try:
+ int_offset -= int(item)
+ except ValueError:
+ symbol_offset += f" - {maybe_parenthesize(item)}"
+ for item in self.pushed:
+ try:
+ int_offset += int(item)
+ except ValueError:
+ symbol_offset += f" + {maybe_parenthesize(item)}"
+ if symbol_offset and not int_offset:
+ res = symbol_offset
+ else:
+ res = f"{int_offset}{symbol_offset}"
+ if res.startswith(" + "):
+ res = res[3:]
+ if res.startswith(" - "):
+ res = "-" + res[3:]
+ return res
+
+ def clear(self) -> None:
+ self.popped = []
+ self.pushed = []
diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py
new file mode 100644
index 0000000..eba9264
--- /dev/null
+++ b/Tools/cases_generator/tier1_generator.py
@@ -0,0 +1,417 @@
+"""Generate the main interpreter switch.
+Reads the instruction definitions from bytecodes.c.
+Writes the cases to generated_cases.c.h, which is #included in ceval.c.
+"""
+
+import argparse
+import os.path
+import sys
+
+from analyzer import (
+ Analysis,
+ Instruction,
+ Uop,
+ Part,
+ analyze_files,
+ Skip,
+ StackItem,
+ analysis_error,
+)
+from cwriter import CWriter
+from typing import TextIO, Iterator
+from lexer import Token
+from stack import StackOffset
+
+
+HERE = os.path.dirname(__file__)
+ROOT = os.path.join(HERE, "../..")
+THIS = os.path.relpath(__file__, ROOT).replace(os.path.sep, "/")
+
+DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c"))
+DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h"))
+
+
+def write_header(filename: str, outfile: TextIO) -> None:
+ outfile.write(
+ f"""// This file is generated by {THIS}
+// from:
+// {filename}
+// Do not edit!
+
+#ifdef TIER_TWO
+ #error "This file is for Tier 1 only"
+#endif
+#define TIER_ONE 1
+"""
+ )
+
+
+FOOTER = "#undef TIER_ONE\n"
+
+
+class SizeMismatch(Exception):
+ pass
+
+
+class Stack:
+ def __init__(self) -> None:
+ self.top_offset = StackOffset()
+ self.base_offset = StackOffset()
+ self.peek_offset = StackOffset()
+ self.variables: list[StackItem] = []
+ self.defined: set[str] = set()
+
+ def pop(self, var: StackItem) -> str:
+ self.top_offset.pop(var)
+ if not var.peek:
+ self.peek_offset.pop(var)
+ indirect = "&" if var.is_array() else ""
+ if self.variables:
+ popped = self.variables.pop()
+ if popped.size != var.size:
+ raise SizeMismatch(
+ f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. "
+ f"Expected {var.size} got {popped.size}"
+ )
+ if popped.name == var.name:
+ return ""
+ elif popped.name == "unused":
+ self.defined.add(var.name)
+ return (
+ f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n"
+ )
+ elif var.name == "unused":
+ return ""
+ else:
+ self.defined.add(var.name)
+ return f"{var.name} = {popped.name};\n"
+ self.base_offset.pop(var)
+ if var.name == "unused":
+ return ""
+ else:
+ self.defined.add(var.name)
+ assign = f"{var.name} = {indirect}stack_pointer[{self.base_offset.to_c()}];"
+ if var.condition:
+ return f"if ({var.condition}) {{ {assign} }}\n"
+ return f"{assign}\n"
+
+ def push(self, var: StackItem) -> str:
+ self.variables.append(var)
+ if var.is_array() and var.name not in self.defined and var.name != "unused":
+ c_offset = self.top_offset.to_c()
+ self.top_offset.push(var)
+ self.defined.add(var.name)
+ return f"{var.name} = &stack_pointer[{c_offset}];\n"
+ else:
+ self.top_offset.push(var)
+ return ""
+
+ def flush(self, out: CWriter) -> None:
+ for var in self.variables:
+ if not var.peek:
+ if var.name != "unused" and not var.is_array():
+ if var.condition:
+ out.emit(f" if ({var.condition}) ")
+ out.emit(
+ f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n"
+ )
+ self.base_offset.push(var)
+ if self.base_offset.to_c() != self.top_offset.to_c():
+ print("base", self.base_offset.to_c(), "top", self.top_offset.to_c())
+ assert False
+ number = self.base_offset.to_c()
+ if number != "0":
+ out.emit(f"stack_pointer += {number};\n")
+ self.variables = []
+ self.base_offset.clear()
+ self.top_offset.clear()
+ self.peek_offset.clear()
+
+ def as_comment(self) -> str:
+ return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */"
+
+
+def declare_variables(inst: Instruction, out: CWriter) -> None:
+ variables = {"unused"}
+ for uop in inst.parts:
+ if isinstance(uop, Uop):
+ for var in reversed(uop.stack.inputs):
+ if var.name not in variables:
+ type = var.type if var.type else "PyObject *"
+ variables.add(var.name)
+ if var.condition:
+ out.emit(f"{type}{var.name} = NULL;\n")
+ else:
+ out.emit(f"{type}{var.name};\n")
+ for var in uop.stack.outputs:
+ if var.name not in variables:
+ variables.add(var.name)
+ type = var.type if var.type else "PyObject *"
+ if var.condition:
+ out.emit(f"{type}{var.name} = NULL;\n")
+ else:
+ out.emit(f"{type}{var.name};\n")
+
+
+def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None:
+ parens = 0
+ for tkn in tkn_iter:
+ if tkn.kind == end and parens == 0:
+ return
+ if tkn.kind == "LPAREN":
+ parens += 1
+ if tkn.kind == "RPAREN":
+ parens -= 1
+ out.emit(tkn)
+
+
+def replace_deopt(
+ out: CWriter,
+ tkn: Token,
+ tkn_iter: Iterator[Token],
+ uop: Uop,
+ unused: Stack,
+ inst: Instruction,
+) -> None:
+ out.emit_at("DEOPT_IF", tkn)
+ out.emit(next(tkn_iter))
+ emit_to(out, tkn_iter, "RPAREN")
+ next(tkn_iter) # Semi colon
+ out.emit(", ")
+ assert inst.family is not None
+ out.emit(inst.family.name)
+ out.emit(");\n")
+
+
+def replace_error(
+ out: CWriter,
+ tkn: Token,
+ tkn_iter: Iterator[Token],
+ uop: Uop,
+ stack: Stack,
+ inst: Instruction,
+) -> None:
+ out.emit_at("if ", tkn)
+ out.emit(next(tkn_iter))
+ emit_to(out, tkn_iter, "COMMA")
+ label = next(tkn_iter).text
+ next(tkn_iter) # RPAREN
+ next(tkn_iter) # Semi colon
+ out.emit(") ")
+ c_offset = stack.peek_offset.to_c()
+ try:
+ offset = -int(c_offset)
+ close = ";\n"
+ except ValueError:
+ offset = None
+ out.emit(f"{{ stack_pointer += {c_offset}; ")
+ close = "; }\n"
+ out.emit("goto ")
+ if offset:
+ out.emit(f"pop_{offset}_")
+ out.emit(label)
+ out.emit(close)
+
+
+def replace_decrefs(
+ out: CWriter,
+ tkn: Token,
+ tkn_iter: Iterator[Token],
+ uop: Uop,
+ stack: Stack,
+ inst: Instruction,
+) -> None:
+ next(tkn_iter)
+ next(tkn_iter)
+ next(tkn_iter)
+ out.emit_at("", tkn)
+ for var in uop.stack.inputs:
+ if var.name == "unused" or var.name == "null" or var.peek:
+ continue
+ if var.size != "1":
+ out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n")
+ out.emit(f"Py_DECREF({var.name}[_i]);\n")
+ out.emit("}\n")
+ elif var.condition:
+ out.emit(f"Py_XDECREF({var.name});\n")
+ else:
+ out.emit(f"Py_DECREF({var.name});\n")
+
+
+def replace_store_sp(
+ out: CWriter,
+ tkn: Token,
+ tkn_iter: Iterator[Token],
+ uop: Uop,
+ stack: Stack,
+ inst: Instruction,
+) -> None:
+ next(tkn_iter)
+ next(tkn_iter)
+ next(tkn_iter)
+ out.emit_at("", tkn)
+ stack.flush(out)
+ out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n")
+
+
+def replace_check_eval_breaker(
+ out: CWriter,
+ tkn: Token,
+ tkn_iter: Iterator[Token],
+ uop: Uop,
+ stack: Stack,
+ inst: Instruction,
+) -> None:
+ next(tkn_iter)
+ next(tkn_iter)
+ next(tkn_iter)
+ if not uop.properties.ends_with_eval_breaker:
+ out.emit_at("CHECK_EVAL_BREAKER();", tkn)
+
+
+REPLACEMENT_FUNCTIONS = {
+ "DEOPT_IF": replace_deopt,
+ "ERROR_IF": replace_error,
+ "DECREF_INPUTS": replace_decrefs,
+ "CHECK_EVAL_BREAKER": replace_check_eval_breaker,
+ "STORE_SP": replace_store_sp,
+}
+
+
+# Move this to formatter
+def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None:
+ tkns = uop.body[1:-1]
+ if not tkns:
+ return
+ tkn_iter = iter(tkns)
+ out.start_line()
+ for tkn in tkn_iter:
+ if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS:
+ REPLACEMENT_FUNCTIONS[tkn.text](out, tkn, tkn_iter, uop, stack, inst)
+ else:
+ out.emit(tkn)
+
+
+def write_uop(
+ uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool
+) -> int:
+ # out.emit(stack.as_comment() + "\n")
+ if isinstance(uop, Skip):
+ entries = "entries" if uop.size > 1 else "entry"
+ out.emit(f"/* Skip {uop.size} cache {entries} */\n")
+ return offset + uop.size
+ try:
+ out.start_line()
+ if braces:
+ out.emit(f"// {uop.name}\n")
+ for var in reversed(uop.stack.inputs):
+ out.emit(stack.pop(var))
+ if braces:
+ out.emit("{\n")
+ if not uop.properties.stores_sp:
+ for i, var in enumerate(uop.stack.outputs):
+ out.emit(stack.push(var))
+ for cache in uop.caches:
+ if cache.name != "unused":
+ if cache.size == 4:
+ type = "PyObject *"
+ reader = "read_obj"
+ else:
+ type = f"uint{cache.size*16}_t "
+ reader = f"read_u{cache.size*16}"
+ out.emit(
+ f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n"
+ )
+ offset += cache.size
+ emit_tokens(out, uop, stack, inst)
+ if uop.properties.stores_sp:
+ for i, var in enumerate(uop.stack.outputs):
+ out.emit(stack.push(var))
+ if braces:
+ out.start_line()
+ out.emit("}\n")
+ # out.emit(stack.as_comment() + "\n")
+ return offset
+ except SizeMismatch as ex:
+ raise analysis_error(ex.args[0], uop.body[0])
+
+
+def uses_this(inst: Instruction) -> bool:
+ if inst.properties.needs_this:
+ return True
+ for uop in inst.parts:
+ if isinstance(uop, Skip):
+ continue
+ for cache in uop.caches:
+ if cache.name != "unused":
+ return True
+ return False
+
+
+def generate_tier1(
+ filenames: str, analysis: Analysis, outfile: TextIO, lines: bool
+) -> None:
+ write_header(filenames, outfile)
+ out = CWriter(outfile, 2, lines)
+ out.emit("\n")
+ for name, inst in sorted(analysis.instructions.items()):
+ needs_this = uses_this(inst)
+ out.emit("\n")
+ out.emit(f"TARGET({name}) {{\n")
+ if needs_this and not inst.is_target:
+ out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n")
+ else:
+ out.emit(f"frame->instr_ptr = next_instr;\n")
+ out.emit(f"next_instr += {inst.size};\n")
+ out.emit(f"INSTRUCTION_STATS({name});\n")
+ if inst.is_target:
+ out.emit(f"PREDICTED({name});\n")
+ if needs_this:
+ out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {inst.size};\n")
+ if inst.family is not None:
+ out.emit(
+ f"static_assert({inst.family.size} == {inst.size-1}"
+ ', "incorrect cache size");\n'
+ )
+ declare_variables(inst, out)
+ offset = 1 # The instruction itself
+ stack = Stack()
+ for part in inst.parts:
+ # Only emit braces if more than one uop
+ offset = write_uop(part, out, offset, stack, inst, len(inst.parts) > 1)
+ out.start_line()
+ if not inst.parts[-1].properties.always_exits:
+ stack.flush(out)
+ if inst.parts[-1].properties.ends_with_eval_breaker:
+ out.emit("CHECK_EVAL_BREAKER();\n")
+ out.emit("DISPATCH();\n")
+ out.start_line()
+ out.emit("}")
+ out.emit("\n")
+ outfile.write(FOOTER)
+
+
+arg_parser = argparse.ArgumentParser(
+ description="Generate the code for the interpreter switch.",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+)
+
+arg_parser.add_argument(
+ "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
+)
+
+arg_parser.add_argument(
+ "-l", "--emit-line-directives", help="Emit #line directives", action="store_true"
+)
+
+arg_parser.add_argument(
+ "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)"
+)
+
+if __name__ == "__main__":
+ args = arg_parser.parse_args()
+ if len(args.input) == 0:
+ args.input.append(DEFAULT_INPUT)
+ data = analyze_files(args.input)
+ with open(args.output, "w") as outfile:
+ generate_tier1(args.input, data, outfile, args.emit_line_directives)
generated by cgit v0.12 at 2025-08-16 16:50:43 (GMT)