gh-104584: Baby steps towards generating and executing traces (#105924)

Added a new, experimental, tracing optimizer and interpreter (a.k.a. "tier 2"). This currently pessimizes, so don't use yet -- this is infrastructure so we can experiment with optimizing passes. To enable it, pass ``-Xuops`` or set ``PYTHONUOPS=1``. To get debug output, set ``PYTHONUOPSDEBUG=N`` where ``N`` is a debug level (0-4, where 0 is no debug output and 4 is excessively verbose).

All of this code is likely to change dramatically before the 3.13 feature freeze. But this is a first step.

1 files changed, 1606 insertions, 0 deletions

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
new file mode 100644
index 0000000..1bdee40
--- /dev/null
+++ b/Python/executor_cases.c.h
@@ -0,0 +1,1606 @@
+// This file is generated by Tools/cases_generator/generate_cases.py
+// from:
+//   Python/bytecodes.c
+// Do not edit!
+
+        case NOP: {
+            break;
+        }
+
+        case LOAD_FAST: {
+            PyObject *value;
+            value = GETLOCAL(oparg);
+            assert(value != NULL);
+            Py_INCREF(value);
+            STACK_GROW(1);
+            stack_pointer[-1] = value;
+            break;
+        }
+
+        case LOAD_FAST_AND_CLEAR: {
+            PyObject *value;
+            value = GETLOCAL(oparg);
+            // do not use SETLOCAL here, it decrefs the old value
+            GETLOCAL(oparg) = NULL;
+            STACK_GROW(1);
+            stack_pointer[-1] = value;
+            break;
+        }
+
+        case LOAD_CONST: {
+            PyObject *value;
+            value = GETITEM(FRAME_CO_CONSTS, oparg);
+            Py_INCREF(value);
+            STACK_GROW(1);
+            stack_pointer[-1] = value;
+            break;
+        }
+
+        case STORE_FAST: {
+            PyObject *value = stack_pointer[-1];
+            SETLOCAL(oparg, value);
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case POP_TOP: {
+            PyObject *value = stack_pointer[-1];
+            Py_DECREF(value);
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case PUSH_NULL: {
+            PyObject *res;
+            res = NULL;
+            STACK_GROW(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case END_SEND: {
+            PyObject *value = stack_pointer[-1];
+            PyObject *receiver = stack_pointer[-2];
+            Py_DECREF(receiver);
+            STACK_SHRINK(1);
+            stack_pointer[-1] = value;
+            break;
+        }
+
+        case UNARY_NEGATIVE: {
+            PyObject *value = stack_pointer[-1];
+            PyObject *res;
+            res = PyNumber_Negative(value);
+            Py_DECREF(value);
+            if (res == NULL) goto pop_1_error;
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case UNARY_NOT: {
+            PyObject *value = stack_pointer[-1];
+            PyObject *res;
+            int err = PyObject_IsTrue(value);
+            Py_DECREF(value);
+            if (err < 0) goto pop_1_error;
+            if (err == 0) {
+                res = Py_True;
+            }
+            else {
+                res = Py_False;
+            }
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case UNARY_INVERT: {
+            PyObject *value = stack_pointer[-1];
+            PyObject *res;
+            res = PyNumber_Invert(value);
+            Py_DECREF(value);
+            if (res == NULL) goto pop_1_error;
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case _GUARD_BOTH_INT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
+            DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
+            break;
+        }
+
+        case _BINARY_OP_MULTIPLY_INT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            STAT_INC(BINARY_OP, hit);
+            res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
+            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
+            if (res == NULL) goto pop_2_error;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case _BINARY_OP_ADD_INT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            STAT_INC(BINARY_OP, hit);
+            res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
+            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
+            if (res == NULL) goto pop_2_error;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case _BINARY_OP_SUBTRACT_INT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            STAT_INC(BINARY_OP, hit);
+            res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
+            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
+            if (res == NULL) goto pop_2_error;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case _GUARD_BOTH_FLOAT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
+            DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
+            break;
+        }
+
+        case _BINARY_OP_MULTIPLY_FLOAT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            STAT_INC(BINARY_OP, hit);
+            double dres =
+                ((PyFloatObject *)left)->ob_fval *
+                ((PyFloatObject *)right)->ob_fval;
+            DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case _BINARY_OP_ADD_FLOAT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            STAT_INC(BINARY_OP, hit);
+            double dres =
+                ((PyFloatObject *)left)->ob_fval +
+                ((PyFloatObject *)right)->ob_fval;
+            DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case _BINARY_OP_SUBTRACT_FLOAT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            STAT_INC(BINARY_OP, hit);
+            double dres =
+                ((PyFloatObject *)left)->ob_fval -
+                ((PyFloatObject *)right)->ob_fval;
+            DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case _GUARD_BOTH_UNICODE: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
+            DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
+            break;
+        }
+
+        case _BINARY_OP_ADD_UNICODE: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            STAT_INC(BINARY_OP, hit);
+            res = PyUnicode_Concat(left, right);
+            _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
+            _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
+            if (res == NULL) goto pop_2_error;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case BINARY_SLICE: {
+            PyObject *stop = stack_pointer[-1];
+            PyObject *start = stack_pointer[-2];
+            PyObject *container = stack_pointer[-3];
+            PyObject *res;
+            PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
+            // Can't use ERROR_IF() here, because we haven't
+            // DECREF'ed container yet, and we still own slice.
+            if (slice == NULL) {
+                res = NULL;
+            }
+            else {
+                res = PyObject_GetItem(container, slice);
+                Py_DECREF(slice);
+            }
+            Py_DECREF(container);
+            if (res == NULL) goto pop_3_error;
+            STACK_SHRINK(2);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case STORE_SLICE: {
+            PyObject *stop = stack_pointer[-1];
+            PyObject *start = stack_pointer[-2];
+            PyObject *container = stack_pointer[-3];
+            PyObject *v = stack_pointer[-4];
+            PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
+            int err;
+            if (slice == NULL) {
+                err = 1;
+            }
+            else {
+                err = PyObject_SetItem(container, slice, v);
+                Py_DECREF(slice);
+            }
+            Py_DECREF(v);
+            Py_DECREF(container);
+            if (err) goto pop_4_error;
+            STACK_SHRINK(4);
+            break;
+        }
+
+        case BINARY_SUBSCR_LIST_INT: {
+            PyObject *sub = stack_pointer[-1];
+            PyObject *list = stack_pointer[-2];
+            PyObject *res;
+            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
+            DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
+
+            // Deopt unless 0 <= sub < PyList_Size(list)
+            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
+            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
+            DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
+            STAT_INC(BINARY_SUBSCR, hit);
+            res = PyList_GET_ITEM(list, index);
+            assert(res != NULL);
+            Py_INCREF(res);
+            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
+            Py_DECREF(list);
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case BINARY_SUBSCR_TUPLE_INT: {
+            PyObject *sub = stack_pointer[-1];
+            PyObject *tuple = stack_pointer[-2];
+            PyObject *res;
+            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
+            DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
+
+            // Deopt unless 0 <= sub < PyTuple_Size(list)
+            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
+            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
+            DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
+            STAT_INC(BINARY_SUBSCR, hit);
+            res = PyTuple_GET_ITEM(tuple, index);
+            assert(res != NULL);
+            Py_INCREF(res);
+            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
+            Py_DECREF(tuple);
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case BINARY_SUBSCR_DICT: {
+            PyObject *sub = stack_pointer[-1];
+            PyObject *dict = stack_pointer[-2];
+            PyObject *res;
+            DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
+            STAT_INC(BINARY_SUBSCR, hit);
+            res = PyDict_GetItemWithError(dict, sub);
+            if (res == NULL) {
+                if (!_PyErr_Occurred(tstate)) {
+                    _PyErr_SetKeyError(sub);
+                }
+                Py_DECREF(dict);
+                Py_DECREF(sub);
+                if (true) goto pop_2_error;
+            }
+            Py_INCREF(res);  // Do this before DECREF'ing dict, sub
+            Py_DECREF(dict);
+            Py_DECREF(sub);
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case LIST_APPEND: {
+            PyObject *v = stack_pointer[-1];
+            PyObject *list = stack_pointer[-(2 + (oparg-1))];
+            if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error;
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case SET_ADD: {
+            PyObject *v = stack_pointer[-1];
+            PyObject *set = stack_pointer[-(2 + (oparg-1))];
+            int err = PySet_Add(set, v);
+            Py_DECREF(v);
+            if (err) goto pop_1_error;
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case STORE_SUBSCR_LIST_INT: {
+            PyObject *sub = stack_pointer[-1];
+            PyObject *list = stack_pointer[-2];
+            PyObject *value = stack_pointer[-3];
+            DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
+            DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
+
+            // Ensure nonnegative, zero-or-one-digit ints.
+            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR);
+            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
+            // Ensure index < len(list)
+            DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
+            STAT_INC(STORE_SUBSCR, hit);
+
+            PyObject *old_value = PyList_GET_ITEM(list, index);
+            PyList_SET_ITEM(list, index, value);
+            assert(old_value != NULL);
+            Py_DECREF(old_value);
+            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
+            Py_DECREF(list);
+            STACK_SHRINK(3);
+            break;
+        }
+
+        case STORE_SUBSCR_DICT: {
+            PyObject *sub = stack_pointer[-1];
+            PyObject *dict = stack_pointer[-2];
+            PyObject *value = stack_pointer[-3];
+            DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
+            STAT_INC(STORE_SUBSCR, hit);
+            int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
+            Py_DECREF(dict);
+            if (err) goto pop_3_error;
+            STACK_SHRINK(3);
+            break;
+        }
+
+        case DELETE_SUBSCR: {
+            PyObject *sub = stack_pointer[-1];
+            PyObject *container = stack_pointer[-2];
+            /* del container[sub] */
+            int err = PyObject_DelItem(container, sub);
+            Py_DECREF(container);
+            Py_DECREF(sub);
+            if (err) goto pop_2_error;
+            STACK_SHRINK(2);
+            break;
+        }
+
+        case CALL_INTRINSIC_1: {
+            PyObject *value = stack_pointer[-1];
+            PyObject *res;
+            assert(oparg <= MAX_INTRINSIC_1);
+            res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value);
+            Py_DECREF(value);
+            if (res == NULL) goto pop_1_error;
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case CALL_INTRINSIC_2: {
+            PyObject *value1 = stack_pointer[-1];
+            PyObject *value2 = stack_pointer[-2];
+            PyObject *res;
+            assert(oparg <= MAX_INTRINSIC_2);
+            res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1);
+            Py_DECREF(value2);
+            Py_DECREF(value1);
+            if (res == NULL) goto pop_2_error;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case GET_AITER: {
+            PyObject *obj = stack_pointer[-1];
+            PyObject *iter;
+            unaryfunc getter = NULL;
+            PyTypeObject *type = Py_TYPE(obj);
+
+            if (type->tp_as_async != NULL) {
+                getter = type->tp_as_async->am_aiter;
+            }
+
+            if (getter == NULL) {
+                _PyErr_Format(tstate, PyExc_TypeError,
+                              "'async for' requires an object with "
+                              "__aiter__ method, got %.100s",
+                              type->tp_name);
+                Py_DECREF(obj);
+                if (true) goto pop_1_error;
+            }
+
+            iter = (*getter)(obj);
+            Py_DECREF(obj);
+            if (iter == NULL) goto pop_1_error;
+
+            if (Py_TYPE(iter)->tp_as_async == NULL ||
+                    Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
+
+                _PyErr_Format(tstate, PyExc_TypeError,
+                              "'async for' received an object from __aiter__ "
+                              "that does not implement __anext__: %.100s",
+                              Py_TYPE(iter)->tp_name);
+                Py_DECREF(iter);
+                if (true) goto pop_1_error;
+            }
+            stack_pointer[-1] = iter;
+            break;
+        }
+
+        case GET_ANEXT: {
+            PyObject *aiter = stack_pointer[-1];
+            PyObject *awaitable;
+            unaryfunc getter = NULL;
+            PyObject *next_iter = NULL;
+            PyTypeObject *type = Py_TYPE(aiter);
+
+            if (PyAsyncGen_CheckExact(aiter)) {
+                awaitable = type->tp_as_async->am_anext(aiter);
+                if (awaitable == NULL) {
+                    goto error;
+                }
+            } else {
+                if (type->tp_as_async != NULL){
+                    getter = type->tp_as_async->am_anext;
+                }
+
+                if (getter != NULL) {
+                    next_iter = (*getter)(aiter);
+                    if (next_iter == NULL) {
+                        goto error;
+                    }
+                }
+                else {
+                    _PyErr_Format(tstate, PyExc_TypeError,
+                                  "'async for' requires an iterator with "
+                                  "__anext__ method, got %.100s",
+                                  type->tp_name);
+                    goto error;
+                }
+
+                awaitable = _PyCoro_GetAwaitableIter(next_iter);
+                if (awaitable == NULL) {
+                    _PyErr_FormatFromCause(
+                        PyExc_TypeError,
+                        "'async for' received an invalid object "
+                        "from __anext__: %.100s",
+                        Py_TYPE(next_iter)->tp_name);
+
+                    Py_DECREF(next_iter);
+                    goto error;
+                } else {
+                    Py_DECREF(next_iter);
+                }
+            }
+            STACK_GROW(1);
+            stack_pointer[-1] = awaitable;
+            break;
+        }
+
+        case GET_AWAITABLE: {
+            PyObject *iterable = stack_pointer[-1];
+            PyObject *iter;
+            iter = _PyCoro_GetAwaitableIter(iterable);
+
+            if (iter == NULL) {
+                format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
+            }
+
+            Py_DECREF(iterable);
+
+            if (iter != NULL && PyCoro_CheckExact(iter)) {
+                PyObject *yf = _PyGen_yf((PyGenObject*)iter);
+                if (yf != NULL) {
+                    /* `iter` is a coroutine object that is being
+                       awaited, `yf` is a pointer to the current awaitable
+                       being awaited on. */
+                    Py_DECREF(yf);
+                    Py_CLEAR(iter);
+                    _PyErr_SetString(tstate, PyExc_RuntimeError,
+                                     "coroutine is being awaited already");
+                    /* The code below jumps to `error` if `iter` is NULL. */
+                }
+            }
+
+            if (iter == NULL) goto pop_1_error;
+            stack_pointer[-1] = iter;
+            break;
+        }
+
+        case POP_EXCEPT: {
+            PyObject *exc_value = stack_pointer[-1];
+            _PyErr_StackItem *exc_info = tstate->exc_info;
+            Py_XSETREF(exc_info->exc_value, exc_value);
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case LOAD_ASSERTION_ERROR: {
+            PyObject *value;
+            value = Py_NewRef(PyExc_AssertionError);
+            STACK_GROW(1);
+            stack_pointer[-1] = value;
+            break;
+        }
+
+        case LOAD_BUILD_CLASS: {
+            PyObject *bc;
+            if (PyDict_CheckExact(BUILTINS())) {
+                bc = _PyDict_GetItemWithError(BUILTINS(),
+                                              &_Py_ID(__build_class__));
+                if (bc == NULL) {
+                    if (!_PyErr_Occurred(tstate)) {
+                        _PyErr_SetString(tstate, PyExc_NameError,
+                                         "__build_class__ not found");
+                    }
+                    if (true) goto error;
+                }
+                Py_INCREF(bc);
+            }
+            else {
+                bc = PyObject_GetItem(BUILTINS(), &_Py_ID(__build_class__));
+                if (bc == NULL) {
+                    if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
+                        _PyErr_SetString(tstate, PyExc_NameError,
+                                         "__build_class__ not found");
+                    if (true) goto error;
+                }
+            }
+            STACK_GROW(1);
+            stack_pointer[-1] = bc;
+            break;
+        }
+
+        case STORE_NAME: {
+            PyObject *v = stack_pointer[-1];
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
+            PyObject *ns = LOCALS();
+            int err;
+            if (ns == NULL) {
+                _PyErr_Format(tstate, PyExc_SystemError,
+                              "no locals found when storing %R", name);
+                Py_DECREF(v);
+                if (true) goto pop_1_error;
+            }
+            if (PyDict_CheckExact(ns))
+                err = PyDict_SetItem(ns, name, v);
+            else
+                err = PyObject_SetItem(ns, name, v);
+            Py_DECREF(v);
+            if (err) goto pop_1_error;
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case DELETE_NAME: {
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
+            PyObject *ns = LOCALS();
+            int err;
+            if (ns == NULL) {
+                _PyErr_Format(tstate, PyExc_SystemError,
+                              "no locals when deleting %R", name);
+                goto error;
+            }
+            err = PyObject_DelItem(ns, name);
+            // Can't use ERROR_IF here.
+            if (err != 0) {
+                format_exc_check_arg(tstate, PyExc_NameError,
+                                     NAME_ERROR_MSG,
+                                     name);
+                goto error;
+            }
+            break;
+        }
+
+        case UNPACK_SEQUENCE_TWO_TUPLE: {
+            PyObject *seq = stack_pointer[-1];
+            PyObject **values = stack_pointer - (1);
+            DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
+            DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE);
+            assert(oparg == 2);
+            STAT_INC(UNPACK_SEQUENCE, hit);
+            values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
+            values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
+            Py_DECREF(seq);
+            STACK_SHRINK(1);
+            STACK_GROW(oparg);
+            break;
+        }
+
+        case UNPACK_SEQUENCE_TUPLE: {
+            PyObject *seq = stack_pointer[-1];
+            PyObject **values = stack_pointer - (1);
+            DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
+            DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
+            STAT_INC(UNPACK_SEQUENCE, hit);
+            PyObject **items = _PyTuple_ITEMS(seq);
+            for (int i = oparg; --i >= 0; ) {
+                *values++ = Py_NewRef(items[i]);
+            }
+            Py_DECREF(seq);
+            STACK_SHRINK(1);
+            STACK_GROW(oparg);
+            break;
+        }
+
+        case UNPACK_SEQUENCE_LIST: {
+            PyObject *seq = stack_pointer[-1];
+            PyObject **values = stack_pointer - (1);
+            DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE);
+            DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
+            STAT_INC(UNPACK_SEQUENCE, hit);
+            PyObject **items = _PyList_ITEMS(seq);
+            for (int i = oparg; --i >= 0; ) {
+                *values++ = Py_NewRef(items[i]);
+            }
+            Py_DECREF(seq);
+            STACK_SHRINK(1);
+            STACK_GROW(oparg);
+            break;
+        }
+
+        case UNPACK_EX: {
+            PyObject *seq = stack_pointer[-1];
+            int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
+            PyObject **top = stack_pointer + totalargs - 1;
+            int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
+            Py_DECREF(seq);
+            if (res == 0) goto pop_1_error;
+            STACK_GROW((oparg & 0xFF) + (oparg >> 8));
+            break;
+        }
+
+        case DELETE_ATTR: {
+            PyObject *owner = stack_pointer[-1];
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
+            int err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
+            Py_DECREF(owner);
+            if (err) goto pop_1_error;
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case STORE_GLOBAL: {
+            PyObject *v = stack_pointer[-1];
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
+            int err = PyDict_SetItem(GLOBALS(), name, v);
+            Py_DECREF(v);
+            if (err) goto pop_1_error;
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case DELETE_GLOBAL: {
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
+            int err;
+            err = PyDict_DelItem(GLOBALS(), name);
+            // Can't use ERROR_IF here.
+            if (err != 0) {
+                if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
+                    format_exc_check_arg(tstate, PyExc_NameError,
+                                         NAME_ERROR_MSG, name);
+                }
+                goto error;
+            }
+            break;
+        }
+
+        case _LOAD_LOCALS: {
+            PyObject *locals;
+            locals = LOCALS();
+            if (locals == NULL) {
+                _PyErr_SetString(tstate, PyExc_SystemError,
+                                 "no locals found");
+                if (true) goto error;
+            }
+            Py_INCREF(locals);
+            STACK_GROW(1);
+            stack_pointer[-1] = locals;
+            break;
+        }
+
+        case _LOAD_FROM_DICT_OR_GLOBALS: {
+            PyObject *mod_or_class_dict = stack_pointer[-1];
+            PyObject *v;
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
+            if (PyDict_CheckExact(mod_or_class_dict)) {
+                v = PyDict_GetItemWithError(mod_or_class_dict, name);
+                if (v != NULL) {
+                    Py_INCREF(v);
+                }
+                else if (_PyErr_Occurred(tstate)) {
+                    Py_DECREF(mod_or_class_dict);
+                    goto error;
+                }
+            }
+            else {
+                v = PyObject_GetItem(mod_or_class_dict, name);
+                if (v == NULL) {
+                    if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
+                        Py_DECREF(mod_or_class_dict);
+                        goto error;
+                    }
+                    _PyErr_Clear(tstate);
+                }
+            }
+            Py_DECREF(mod_or_class_dict);
+            if (v == NULL) {
+                v = PyDict_GetItemWithError(GLOBALS(), name);
+                if (v != NULL) {
+                    Py_INCREF(v);
+                }
+                else if (_PyErr_Occurred(tstate)) {
+                    goto error;
+                }
+                else {
+                    if (PyDict_CheckExact(BUILTINS())) {
+                        v = PyDict_GetItemWithError(BUILTINS(), name);
+                        if (v == NULL) {
+                            if (!_PyErr_Occurred(tstate)) {
+                                format_exc_check_arg(
+                                        tstate, PyExc_NameError,
+                                        NAME_ERROR_MSG, name);
+                            }
+                            goto error;
+                        }
+                        Py_INCREF(v);
+                    }
+                    else {
+                        v = PyObject_GetItem(BUILTINS(), name);
+                        if (v == NULL) {
+                            if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
+                                format_exc_check_arg(
+                                            tstate, PyExc_NameError,
+                                            NAME_ERROR_MSG, name);
+                            }
+                            goto error;
+                        }
+                    }
+                }
+            }
+            stack_pointer[-1] = v;
+            break;
+        }
+
+        case DELETE_DEREF: {
+            PyObject *cell = GETLOCAL(oparg);
+            PyObject *oldobj = PyCell_GET(cell);
+            // Can't use ERROR_IF here.
+            // Fortunately we don't need its superpower.
+            if (oldobj == NULL) {
+                format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+                goto error;
+            }
+            PyCell_SET(cell, NULL);
+            Py_DECREF(oldobj);
+            break;
+        }
+
+        case LOAD_FROM_DICT_OR_DEREF: {
+            PyObject *class_dict = stack_pointer[-1];
+            PyObject *value;
+            PyObject *name;
+            assert(class_dict);
+            assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
+            name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
+            if (PyDict_CheckExact(class_dict)) {
+                value = PyDict_GetItemWithError(class_dict, name);
+                if (value != NULL) {
+                    Py_INCREF(value);
+                }
+                else if (_PyErr_Occurred(tstate)) {
+                    Py_DECREF(class_dict);
+                    goto error;
+                }
+            }
+            else {
+                value = PyObject_GetItem(class_dict, name);
+                if (value == NULL) {
+                    if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
+                        Py_DECREF(class_dict);
+                        goto error;
+                    }
+                    _PyErr_Clear(tstate);
+                }
+            }
+            Py_DECREF(class_dict);
+            if (!value) {
+                PyObject *cell = GETLOCAL(oparg);
+                value = PyCell_GET(cell);
+                if (value == NULL) {
+                    format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+                    goto error;
+                }
+                Py_INCREF(value);
+            }
+            stack_pointer[-1] = value;
+            break;
+        }
+
+        case LOAD_DEREF: {
+            PyObject *value;
+            PyObject *cell = GETLOCAL(oparg);
+            value = PyCell_GET(cell);
+            if (value == NULL) {
+                format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+                if (true) goto error;
+            }
+            Py_INCREF(value);
+            STACK_GROW(1);
+            stack_pointer[-1] = value;
+            break;
+        }
+
+        case STORE_DEREF: {
+            PyObject *v = stack_pointer[-1];
+            PyObject *cell = GETLOCAL(oparg);
+            PyObject *oldobj = PyCell_GET(cell);
+            PyCell_SET(cell, v);
+            Py_XDECREF(oldobj);
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case COPY_FREE_VARS: {
+            /* Copy closure variables to free variables */
+            PyCodeObject *co = _PyFrame_GetCode(frame);
+            assert(PyFunction_Check(frame->f_funcobj));
+            PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure;
+            assert(oparg == co->co_nfreevars);
+            int offset = co->co_nlocalsplus - oparg;
+            for (int i = 0; i < oparg; ++i) {
+                PyObject *o = PyTuple_GET_ITEM(closure, i);
+                frame->localsplus[offset + i] = Py_NewRef(o);
+            }
+            break;
+        }
+
+        case BUILD_STRING: {
+            PyObject **pieces = (stack_pointer - oparg);
+            PyObject *str;
+            str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
+            for (int _i = oparg; --_i >= 0;) {
+                Py_DECREF(pieces[_i]);
+            }
+            if (str == NULL) { STACK_SHRINK(oparg); goto error; }
+            STACK_SHRINK(oparg);
+            STACK_GROW(1);
+            stack_pointer[-1] = str;
+            break;
+        }
+
+        case BUILD_TUPLE: {
+            PyObject **values = (stack_pointer - oparg);
+            PyObject *tup;
+            tup = _PyTuple_FromArraySteal(values, oparg);
+            if (tup == NULL) { STACK_SHRINK(oparg); goto error; }
+            STACK_SHRINK(oparg);
+            STACK_GROW(1);
+            stack_pointer[-1] = tup;
+            break;
+        }
+
+        case BUILD_LIST: {
+            PyObject **values = (stack_pointer - oparg);
+            PyObject *list;
+            list = _PyList_FromArraySteal(values, oparg);
+            if (list == NULL) { STACK_SHRINK(oparg); goto error; }
+            STACK_SHRINK(oparg);
+            STACK_GROW(1);
+            stack_pointer[-1] = list;
+            break;
+        }
+
+        case LIST_EXTEND: {
+            PyObject *iterable = stack_pointer[-1];
+            PyObject *list = stack_pointer[-(2 + (oparg-1))];
+            PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
+            if (none_val == NULL) {
+                if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
+                   (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
+                {
+                    _PyErr_Clear(tstate);
+                    _PyErr_Format(tstate, PyExc_TypeError,
+                          "Value after * must be an iterable, not %.200s",
+                          Py_TYPE(iterable)->tp_name);
+                }
+                Py_DECREF(iterable);
+                if (true) goto pop_1_error;
+            }
+            assert(Py_IsNone(none_val));
+            Py_DECREF(iterable);
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case SET_UPDATE: {
+            PyObject *iterable = stack_pointer[-1];
+            PyObject *set = stack_pointer[-(2 + (oparg-1))];
+            int err = _PySet_Update(set, iterable);
+            Py_DECREF(iterable);
+            if (err < 0) goto pop_1_error;
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case BUILD_SET: {
+            PyObject **values = (stack_pointer - oparg);
+            PyObject *set;
+            set = PySet_New(NULL);
+            if (set == NULL)
+                goto error;
+            int err = 0;
+            for (int i = 0; i < oparg; i++) {
+                PyObject *item = values[i];
+                if (err == 0)
+                    err = PySet_Add(set, item);
+                Py_DECREF(item);
+            }
+            if (err != 0) {
+                Py_DECREF(set);
+                if (true) { STACK_SHRINK(oparg); goto error; }
+            }
+            STACK_SHRINK(oparg);
+            STACK_GROW(1);
+            stack_pointer[-1] = set;
+            break;
+        }
+
+        case BUILD_MAP: {
+            PyObject **values = (stack_pointer - oparg*2);
+            PyObject *map;
+            map = _PyDict_FromItems(
+                    values, 2,
+                    values+1, 2,
+                    oparg);
+            if (map == NULL)
+                goto error;
+
+            for (int _i = oparg*2; --_i >= 0;) {
+                Py_DECREF(values[_i]);
+            }
+            if (map == NULL) { STACK_SHRINK(oparg*2); goto error; }
+            STACK_SHRINK(oparg*2);
+            STACK_GROW(1);
+            stack_pointer[-1] = map;
+            break;
+        }
+
+        case SETUP_ANNOTATIONS: {
+            int err;
+            PyObject *ann_dict;
+            if (LOCALS() == NULL) {
+                _PyErr_Format(tstate, PyExc_SystemError,
+                              "no locals found when setting up annotations");
+                if (true) goto error;
+            }
+            /* check if __annotations__ in locals()... */
+            if (PyDict_CheckExact(LOCALS())) {
+                ann_dict = _PyDict_GetItemWithError(LOCALS(),
+                                                    &_Py_ID(__annotations__));
+                if (ann_dict == NULL) {
+                    if (_PyErr_Occurred(tstate)) goto error;
+                    /* ...if not, create a new one */
+                    ann_dict = PyDict_New();
+                    if (ann_dict == NULL) goto error;
+                    err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__),
+                                         ann_dict);
+                    Py_DECREF(ann_dict);
+                    if (err) goto error;
+                }
+            }
+            else {
+                /* do the same if locals() is not a dict */
+                ann_dict = PyObject_GetItem(LOCALS(), &_Py_ID(__annotations__));
+                if (ann_dict == NULL) {
+                    if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) goto error;
+                    _PyErr_Clear(tstate);
+                    ann_dict = PyDict_New();
+                    if (ann_dict == NULL) goto error;
+                    err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
+                                           ann_dict);
+                    Py_DECREF(ann_dict);
+                    if (err) goto error;
+                }
+                else {
+                    Py_DECREF(ann_dict);
+                }
+            }
+            break;
+        }
+
+        case BUILD_CONST_KEY_MAP: {
+            PyObject *keys = stack_pointer[-1];
+            PyObject **values = (stack_pointer - (1 + oparg));
+            PyObject *map;
+            if (!PyTuple_CheckExact(keys) ||
+                PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
+                _PyErr_SetString(tstate, PyExc_SystemError,
+                                 "bad BUILD_CONST_KEY_MAP keys argument");
+                goto error;  // Pop the keys and values.
+            }
+            map = _PyDict_FromItems(
+                    &PyTuple_GET_ITEM(keys, 0), 1,
+                    values, 1, oparg);
+            for (int _i = oparg; --_i >= 0;) {
+                Py_DECREF(values[_i]);
+            }
+            Py_DECREF(keys);
+            if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; }
+            STACK_SHRINK(oparg);
+            stack_pointer[-1] = map;
+            break;
+        }
+
+        case DICT_UPDATE: {
+            PyObject *update = stack_pointer[-1];
+            PyObject *dict = PEEK(oparg + 1);  // update is still on the stack
+            if (PyDict_Update(dict, update) < 0) {
+                if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
+                    _PyErr_Format(tstate, PyExc_TypeError,
+                                    "'%.200s' object is not a mapping",
+                                    Py_TYPE(update)->tp_name);
+                }
+                Py_DECREF(update);
+                if (true) goto pop_1_error;
+            }
+            Py_DECREF(update);
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case DICT_MERGE: {
+            PyObject *update = stack_pointer[-1];
+            PyObject *dict = PEEK(oparg + 1);  // update is still on the stack
+
+            if (_PyDict_MergeEx(dict, update, 2) < 0) {
+                format_kwargs_error(tstate, PEEK(3 + oparg), update);
+                Py_DECREF(update);
+                if (true) goto pop_1_error;
+            }
+            Py_DECREF(update);
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case MAP_ADD: {
+            PyObject *value = stack_pointer[-1];
+            PyObject *key = stack_pointer[-2];
+            PyObject *dict = PEEK(oparg + 2);  // key, value are still on the stack
+            assert(PyDict_CheckExact(dict));
+            /* dict[key] = value */
+            // Do not DECREF INPUTS because the function steals the references
+            if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error;
+            STACK_SHRINK(2);
+            break;
+        }
+
+        case LOAD_SUPER_ATTR_ATTR: {
+            PyObject *self = stack_pointer[-1];
+            PyObject *class = stack_pointer[-2];
+            PyObject *global_super = stack_pointer[-3];
+            PyObject *res2 = NULL;
+            PyObject *res;
+            assert(!(oparg & 1));
+            DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
+            DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
+            STAT_INC(LOAD_SUPER_ATTR, hit);
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
+            res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
+            Py_DECREF(global_super);
+            Py_DECREF(class);
+            Py_DECREF(self);
+            if (res == NULL) goto pop_3_error;
+            STACK_SHRINK(2);
+            STACK_GROW(((oparg & 1) ? 1 : 0));
+            stack_pointer[-1] = res;
+            if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; }
+            break;
+        }
+
+        case LOAD_SUPER_ATTR_METHOD: {
+            PyObject *self = stack_pointer[-1];
+            PyObject *class = stack_pointer[-2];
+            PyObject *global_super = stack_pointer[-3];
+            PyObject *res2;
+            PyObject *res;
+            assert(oparg & 1);
+            DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
+            DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
+            STAT_INC(LOAD_SUPER_ATTR, hit);
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
+            PyTypeObject *cls = (PyTypeObject *)class;
+            int method_found = 0;
+            res2 = _PySuper_Lookup(cls, self, name,
+                                   cls->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
+            Py_DECREF(global_super);
+            Py_DECREF(class);
+            if (res2 == NULL) {
+                Py_DECREF(self);
+                if (true) goto pop_3_error;
+            }
+            if (method_found) {
+                res = self; // transfer ownership
+            } else {
+                Py_DECREF(self);
+                res = res2;
+                res2 = NULL;
+            }
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            stack_pointer[-2] = res2;
+            break;
+        }
+
+        case COMPARE_OP_FLOAT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
+            DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
+            STAT_INC(COMPARE_OP, hit);
+            double dleft = PyFloat_AS_DOUBLE(left);
+            double dright = PyFloat_AS_DOUBLE(right);
+            // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg
+            int sign_ish = COMPARISON_BIT(dleft, dright);
+            _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
+            _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
+            res = (sign_ish & oparg) ? Py_True : Py_False;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case COMPARE_OP_INT: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
+            DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
+            DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP);
+            DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP);
+            STAT_INC(COMPARE_OP, hit);
+            assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 &&
+                   _PyLong_DigitCount((PyLongObject *)right) <= 1);
+            Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left);
+            Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right);
+            // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
+            int sign_ish = COMPARISON_BIT(ileft, iright);
+            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
+            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
+            res = (sign_ish & oparg) ? Py_True : Py_False;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case COMPARE_OP_STR: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *res;
+            DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
+            DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
+            STAT_INC(COMPARE_OP, hit);
+            int eq = _PyUnicode_Equal(left, right);
+            assert((oparg >>4) == Py_EQ || (oparg >>4) == Py_NE);
+            _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
+            _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
+            assert(eq == 0 || eq == 1);
+            assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS);
+            assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
+            res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case IS_OP: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *b;
+            int res = Py_Is(left, right) ^ oparg;
+            Py_DECREF(left);
+            Py_DECREF(right);
+            b = res ? Py_True : Py_False;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = b;
+            break;
+        }
+
+        case CONTAINS_OP: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *b;
+            int res = PySequence_Contains(right, left);
+            Py_DECREF(left);
+            Py_DECREF(right);
+            if (res < 0) goto pop_2_error;
+            b = (res ^ oparg) ? Py_True : Py_False;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = b;
+            break;
+        }
+
+        case CHECK_EG_MATCH: {
+            PyObject *match_type = stack_pointer[-1];
+            PyObject *exc_value = stack_pointer[-2];
+            PyObject *rest;
+            PyObject *match;
+            if (check_except_star_type_valid(tstate, match_type) < 0) {
+                Py_DECREF(exc_value);
+                Py_DECREF(match_type);
+                if (true) goto pop_2_error;
+            }
+
+            match = NULL;
+            rest = NULL;
+            int res = exception_group_match(exc_value, match_type,
+                                            &match, &rest);
+            Py_DECREF(exc_value);
+            Py_DECREF(match_type);
+            if (res < 0) goto pop_2_error;
+
+            assert((match == NULL) == (rest == NULL));
+            if (match == NULL) goto pop_2_error;
+
+            if (!Py_IsNone(match)) {
+                PyErr_SetHandledException(match);
+            }
+            stack_pointer[-1] = match;
+            stack_pointer[-2] = rest;
+            break;
+        }
+
+        case CHECK_EXC_MATCH: {
+            PyObject *right = stack_pointer[-1];
+            PyObject *left = stack_pointer[-2];
+            PyObject *b;
+            assert(PyExceptionInstance_Check(left));
+            if (check_except_type_valid(tstate, right) < 0) {
+                 Py_DECREF(right);
+                 if (true) goto pop_1_error;
+            }
+
+            int res = PyErr_GivenExceptionMatches(left, right);
+            Py_DECREF(right);
+            b = res ? Py_True : Py_False;
+            stack_pointer[-1] = b;
+            break;
+        }
+
+        case GET_LEN: {
+            PyObject *obj = stack_pointer[-1];
+            PyObject *len_o;
+            // PUSH(len(TOS))
+            Py_ssize_t len_i = PyObject_Length(obj);
+            if (len_i < 0) goto error;
+            len_o = PyLong_FromSsize_t(len_i);
+            if (len_o == NULL) goto error;
+            STACK_GROW(1);
+            stack_pointer[-1] = len_o;
+            break;
+        }
+
+        case MATCH_CLASS: {
+            PyObject *names = stack_pointer[-1];
+            PyObject *type = stack_pointer[-2];
+            PyObject *subject = stack_pointer[-3];
+            PyObject *attrs;
+            // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
+            // None on failure.
+            assert(PyTuple_CheckExact(names));
+            attrs = match_class(tstate, subject, type, oparg, names);
+            Py_DECREF(subject);
+            Py_DECREF(type);
+            Py_DECREF(names);
+            if (attrs) {
+                assert(PyTuple_CheckExact(attrs));  // Success!
+            }
+            else {
+                if (_PyErr_Occurred(tstate)) goto pop_3_error;
+                attrs = Py_None;  // Failure!
+            }
+            STACK_SHRINK(2);
+            stack_pointer[-1] = attrs;
+            break;
+        }
+
+        case MATCH_MAPPING: {
+            PyObject *subject = stack_pointer[-1];
+            PyObject *res;
+            int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
+            res = match ? Py_True : Py_False;
+            STACK_GROW(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case MATCH_SEQUENCE: {
+            PyObject *subject = stack_pointer[-1];
+            PyObject *res;
+            int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
+            res = match ? Py_True : Py_False;
+            STACK_GROW(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case MATCH_KEYS: {
+            PyObject *keys = stack_pointer[-1];
+            PyObject *subject = stack_pointer[-2];
+            PyObject *values_or_none;
+            // On successful match, PUSH(values). Otherwise, PUSH(None).
+            values_or_none = match_keys(tstate, subject, keys);
+            if (values_or_none == NULL) goto error;
+            STACK_GROW(1);
+            stack_pointer[-1] = values_or_none;
+            break;
+        }
+
+        case GET_ITER: {
+            PyObject *iterable = stack_pointer[-1];
+            PyObject *iter;
+            /* before: [obj]; after [getiter(obj)] */
+            iter = PyObject_GetIter(iterable);
+            Py_DECREF(iterable);
+            if (iter == NULL) goto pop_1_error;
+            stack_pointer[-1] = iter;
+            break;
+        }
+
+        case GET_YIELD_FROM_ITER: {
+            PyObject *iterable = stack_pointer[-1];
+            PyObject *iter;
+            /* before: [obj]; after [getiter(obj)] */
+            if (PyCoro_CheckExact(iterable)) {
+                /* `iterable` is a coroutine */
+                if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
+                    /* and it is used in a 'yield from' expression of a
+                       regular generator. */
+                    _PyErr_SetString(tstate, PyExc_TypeError,
+                                     "cannot 'yield from' a coroutine object "
+                                     "in a non-coroutine generator");
+                    goto error;
+                }
+                iter = iterable;
+            }
+            else if (PyGen_CheckExact(iterable)) {
+                iter = iterable;
+            }
+            else {
+                /* `iterable` is not a generator. */
+                iter = PyObject_GetIter(iterable);
+                if (iter == NULL) {
+                    goto error;
+                }
+                Py_DECREF(iterable);
+            }
+            stack_pointer[-1] = iter;
+            break;
+        }
+
+        case WITH_EXCEPT_START: {
+            PyObject *val = stack_pointer[-1];
+            PyObject *lasti = stack_pointer[-3];
+            PyObject *exit_func = stack_pointer[-4];
+            PyObject *res;
+            /* At the top of the stack are 4 values:
+               - val: TOP = exc_info()
+               - unused: SECOND = previous exception
+               - lasti: THIRD = lasti of exception in exc_info()
+               - exit_func: FOURTH = the context.__exit__ bound method
+               We call FOURTH(type(TOP), TOP, GetTraceback(TOP)).
+               Then we push the __exit__ return value.
+            */
+            PyObject *exc, *tb;
+
+            assert(val && PyExceptionInstance_Check(val));
+            exc = PyExceptionInstance_Class(val);
+            tb = PyException_GetTraceback(val);
+            Py_XDECREF(tb);
+            assert(PyLong_Check(lasti));
+            (void)lasti; // Shut up compiler warning if asserts are off
+            PyObject *stack[4] = {NULL, exc, val, tb};
+            res = PyObject_Vectorcall(exit_func, stack + 1,
+                    3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
+            if (res == NULL) goto error;
+            STACK_GROW(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case PUSH_EXC_INFO: {
+            PyObject *new_exc = stack_pointer[-1];
+            PyObject *prev_exc;
+            _PyErr_StackItem *exc_info = tstate->exc_info;
+            if (exc_info->exc_value != NULL) {
+                prev_exc = exc_info->exc_value;
+            }
+            else {
+                prev_exc = Py_None;
+            }
+            assert(PyExceptionInstance_Check(new_exc));
+            exc_info->exc_value = Py_NewRef(new_exc);
+            STACK_GROW(1);
+            stack_pointer[-1] = new_exc;
+            stack_pointer[-2] = prev_exc;
+            break;
+        }
+
+        case EXIT_INIT_CHECK: {
+            PyObject *should_be_none = stack_pointer[-1];
+            assert(STACK_LEVEL() == 2);
+            if (should_be_none != Py_None) {
+                PyErr_Format(PyExc_TypeError,
+                    "__init__() should return None, not '%.200s'",
+                    Py_TYPE(should_be_none)->tp_name);
+                goto error;
+            }
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case MAKE_FUNCTION: {
+            PyObject *codeobj = stack_pointer[-1];
+            PyObject *func;
+
+            PyFunctionObject *func_obj = (PyFunctionObject *)
+                PyFunction_New(codeobj, GLOBALS());
+
+            Py_DECREF(codeobj);
+            if (func_obj == NULL) {
+                goto error;
+            }
+
+            func_obj->func_version = ((PyCodeObject *)codeobj)->co_version;
+            func = (PyObject *)func_obj;
+            stack_pointer[-1] = func;
+            break;
+        }
+
+        case SET_FUNCTION_ATTRIBUTE: {
+            PyObject *func = stack_pointer[-1];
+            PyObject *attr = stack_pointer[-2];
+            assert(PyFunction_Check(func));
+            PyFunctionObject *func_obj = (PyFunctionObject *)func;
+            switch(oparg) {
+                case MAKE_FUNCTION_CLOSURE:
+                    assert(func_obj->func_closure == NULL);
+                    func_obj->func_closure = attr;
+                    break;
+                case MAKE_FUNCTION_ANNOTATIONS:
+                    assert(func_obj->func_annotations == NULL);
+                    func_obj->func_annotations = attr;
+                    break;
+                case MAKE_FUNCTION_KWDEFAULTS:
+                    assert(PyDict_CheckExact(attr));
+                    assert(func_obj->func_kwdefaults == NULL);
+                    func_obj->func_kwdefaults = attr;
+                    break;
+                case MAKE_FUNCTION_DEFAULTS:
+                    assert(PyTuple_CheckExact(attr));
+                    assert(func_obj->func_defaults == NULL);
+                    func_obj->func_defaults = attr;
+                    break;
+                default:
+                    Py_UNREACHABLE();
+            }
+            STACK_SHRINK(1);
+            stack_pointer[-1] = func;
+            break;
+        }
+
+        case BUILD_SLICE: {
+            PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL;
+            PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))];
+            PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))];
+            PyObject *slice;
+            slice = PySlice_New(start, stop, step);
+            Py_DECREF(start);
+            Py_DECREF(stop);
+            Py_XDECREF(step);
+            if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; }
+            STACK_SHRINK(((oparg == 3) ? 1 : 0));
+            STACK_SHRINK(1);
+            stack_pointer[-1] = slice;
+            break;
+        }
+
+        case CONVERT_VALUE: {
+            PyObject *value = stack_pointer[-1];
+            PyObject *result;
+            convertion_func_ptr  conv_fn;
+            assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
+            conv_fn = CONVERSION_FUNCTIONS[oparg];
+            result = conv_fn(value);
+            Py_DECREF(value);
+            if (result == NULL) goto pop_1_error;
+            stack_pointer[-1] = result;
+            break;
+        }
+
+        case FORMAT_SIMPLE: {
+            PyObject *value = stack_pointer[-1];
+            PyObject *res;
+            /* If value is a unicode object, then we know the result
+             * of format(value) is value itself. */
+            if (!PyUnicode_CheckExact(value)) {
+                res = PyObject_Format(value, NULL);
+                Py_DECREF(value);
+                if (res == NULL) goto pop_1_error;
+            }
+            else {
+                res = value;
+            }
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case FORMAT_WITH_SPEC: {
+            PyObject *fmt_spec = stack_pointer[-1];
+            PyObject *value = stack_pointer[-2];
+            PyObject *res;
+            res = PyObject_Format(value, fmt_spec);
+            Py_DECREF(value);
+            Py_DECREF(fmt_spec);
+            if (res == NULL) goto pop_2_error;
+            STACK_SHRINK(1);
+            stack_pointer[-1] = res;
+            break;
+        }
+
+        case COPY: {
+            PyObject *bottom = stack_pointer[-(1 + (oparg-1))];
+            PyObject *top;
+            assert(oparg > 0);
+            top = Py_NewRef(bottom);
+            STACK_GROW(1);
+            stack_pointer[-1] = top;
+            break;
+        }
+
+        case SWAP: {
+            PyObject *top = stack_pointer[-1];
+            PyObject *bottom = stack_pointer[-(2 + (oparg-2))];
+            assert(oparg >= 2);
+            stack_pointer[-1] = bottom;
+            stack_pointer[-(2 + (oparg-2))] = top;
+            break;
+        }