bpo-46841: Quicken code in-place (GH-31888)

* Moves the bytecode to the end of the corresponding PyCodeObject, and quickens it in-place.

* Removes the almost-always-unused co_varnames, co_freevars, and co_cellvars member caches

* _PyOpcode_Deopt is a new mapping from all opcodes to their un-quickened forms.

* _PyOpcode_InlineCacheEntries is renamed to _PyOpcode_Caches

* _Py_IncrementCountAndMaybeQuicken is renamed to _PyCode_Warmup

* _Py_Quicken is renamed to _PyCode_Quicken

* _co_quickened is renamed to _co_code_adaptive (and is now a read-only memoryview).

* Do not emit unused nonzero opargs anymore in the compiler.

5 files changed, 149 insertions, 137 deletions

diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h
index ee425f6..272bcd6 100644
--- a/Objects/clinic/codeobject.c.h
+++ b/Objects/clinic/codeobject.c.h
@@ -203,12 +203,12 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
     int co_stacksize = self->co_stacksize;
     int co_flags = self->co_flags;
     int co_firstlineno = self->co_firstlineno;
-    PyBytesObject *co_code = (PyBytesObject *)self->co_code;
+    PyBytesObject *co_code = NULL;
     PyObject *co_consts = self->co_consts;
     PyObject *co_names = self->co_names;
-    PyObject *co_varnames = self->co_varnames;
-    PyObject *co_freevars = self->co_freevars;
-    PyObject *co_cellvars = self->co_cellvars;
+    PyObject *co_varnames = NULL;
+    PyObject *co_freevars = NULL;
+    PyObject *co_cellvars = NULL;
     PyObject *co_filename = self->co_filename;
     PyObject *co_name = self->co_name;
     PyObject *co_qualname = self->co_qualname;
@@ -456,4 +456,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=9e8c4a19474ec520 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=b1b83a70ffc5b7cd input=a9049054013a1b77]*/
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index 5279f6c..224493e 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -305,9 +305,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
     co->co_qualname = con->qualname;
     co->co_flags = con->flags;
 
-    Py_INCREF(con->code);
-    co->co_code = con->code;
-    co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(con->code);
     co->co_firstlineno = con->firstlineno;
     Py_INCREF(con->linetable);
     co->co_linetable = con->linetable;
@@ -341,16 +338,14 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
     co->co_nplaincellvars = nplaincellvars;
     co->co_ncellvars = ncellvars;
     co->co_nfreevars = nfreevars;
-    co->co_varnames = NULL;
-    co->co_cellvars = NULL;
-    co->co_freevars = NULL;
 
     /* not set */
     co->co_weakreflist = NULL;
     co->co_extra = NULL;
 
     co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
-    co->co_quickened = NULL;
+    memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code),
+           PyBytes_GET_SIZE(con->code));
 }
 
 /* The caller is responsible for ensuring that the given data is valid. */
@@ -386,7 +381,8 @@ _PyCode_New(struct _PyCodeConstructor *con)
         con->columntable = Py_None;
     }
 
-    PyCodeObject *co = PyObject_New(PyCodeObject, &PyCode_Type);
+    Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT);
+    PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size);
     if (co == NULL) {
         PyErr_NoMemory();
         return NULL;
@@ -521,13 +517,6 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
         goto error;
     }
 
-    Py_INCREF(varnames);
-    co->co_varnames = varnames;
-    Py_INCREF(cellvars);
-    co->co_cellvars = cellvars;
-    Py_INCREF(freevars);
-    co->co_freevars = freevars;
-
 error:
     Py_XDECREF(localsplusnames);
     Py_XDECREF(localspluskinds);
@@ -611,7 +600,7 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq)
     if (addrq < 0) {
         return co->co_firstlineno;
     }
-    assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code));
+    assert(addrq >= 0 && addrq < _PyCode_NBYTES(co));
     PyCodeAddressRange bounds;
     _PyCode_InitAddressRange(co, &bounds);
     return _PyCode_CheckLineNumber(addrq, &bounds);
@@ -639,7 +628,7 @@ _PyCode_Addr2EndLine(PyCodeObject* co, int addrq)
         return -1;
     }
 
-    assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code));
+    assert(addrq >= 0 && addrq < _PyCode_NBYTES(co));
     PyCodeAddressRange bounds;
     _PyCode_InitEndAddressRange(co, &bounds);
     return _PyCode_CheckLineNumber(addrq, &bounds);
@@ -995,7 +984,7 @@ _source_offset_converter(int* value) {
 static PyObject*
 positionsiter_next(positionsiterator* pi)
 {
-    if (pi->pi_offset >= PyBytes_GET_SIZE(pi->pi_code->co_code)) {
+    if (pi->pi_offset >= _PyCode_NBYTES(pi->pi_code)) {
         return NULL;
     }
 
@@ -1151,46 +1140,39 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
 PyObject *
 _PyCode_GetVarnames(PyCodeObject *co)
 {
-    if (co->co_varnames == NULL) {
-        // PyCodeObject owns this reference.
-        co->co_varnames = get_localsplus_names(co, CO_FAST_LOCAL,
-                                               co->co_nlocals);
-        if (co->co_varnames == NULL) {
-            return NULL;
-        }
-    }
-    Py_INCREF(co->co_varnames);
-    return co->co_varnames;
+    return get_localsplus_names(co, CO_FAST_LOCAL, co->co_nlocals);
 }
 
 PyObject *
 _PyCode_GetCellvars(PyCodeObject *co)
 {
-    if (co->co_cellvars == NULL) {
-        // PyCodeObject owns this reference.
-        co->co_cellvars = get_localsplus_names(co, CO_FAST_CELL,
-                                               co->co_ncellvars);
-        if (co->co_cellvars == NULL) {
-            return NULL;
-        }
-    }
-    Py_INCREF(co->co_cellvars);
-    return co->co_cellvars;
+    return get_localsplus_names(co, CO_FAST_CELL, co->co_ncellvars);
 }
 
 PyObject *
 _PyCode_GetFreevars(PyCodeObject *co)
 {
-    if (co->co_freevars == NULL) {
-        // PyCodeObject owns this reference.
-        co->co_freevars = get_localsplus_names(co, CO_FAST_FREE,
-                                               co->co_nfreevars);
-        if (co->co_freevars == NULL) {
-            return NULL;
+    return get_localsplus_names(co, CO_FAST_FREE, co->co_nfreevars);
+}
+
+PyObject *
+_PyCode_GetCode(PyCodeObject *co)
+{
+    PyObject *code = PyBytes_FromStringAndSize(NULL, _PyCode_NBYTES(co));
+    if (code == NULL) {
+        return NULL;
+    }
+    _Py_CODEUNIT *instructions = (_Py_CODEUNIT *)PyBytes_AS_STRING(code);
+    for (int i = 0; i < Py_SIZE(co); i++) {
+        _Py_CODEUNIT instruction = _PyCode_CODE(co)[i];
+        int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
+        int caches = _PyOpcode_Caches[opcode];
+        instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction));
+        while (caches--) {
+            instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0);
         }
     }
-    Py_INCREF(co->co_freevars);
-    return co->co_freevars;
+    return code;
 }
 
 
@@ -1348,14 +1330,10 @@ code_dealloc(PyCodeObject *co)
         PyMem_Free(co_extra);
     }
 
-    Py_XDECREF(co->co_code);
     Py_XDECREF(co->co_consts);
     Py_XDECREF(co->co_names);
     Py_XDECREF(co->co_localsplusnames);
     Py_XDECREF(co->co_localspluskinds);
-    Py_XDECREF(co->co_varnames);
-    Py_XDECREF(co->co_freevars);
-    Py_XDECREF(co->co_cellvars);
     Py_XDECREF(co->co_filename);
     Py_XDECREF(co->co_name);
     Py_XDECREF(co->co_qualname);
@@ -1363,10 +1341,10 @@ code_dealloc(PyCodeObject *co)
     Py_XDECREF(co->co_endlinetable);
     Py_XDECREF(co->co_columntable);
     Py_XDECREF(co->co_exceptiontable);
-    if (co->co_weakreflist != NULL)
+    if (co->co_weakreflist != NULL) {
         PyObject_ClearWeakRefs((PyObject*)co);
-    if (co->co_quickened) {
-        PyMem_Free(co->co_quickened);
+    }
+    if (co->co_warmup == 0) {
         _Py_QuickenedCount--;
     }
     PyObject_Free(co);
@@ -1420,8 +1398,21 @@ code_richcompare(PyObject *self, PyObject *other, int op)
     if (!eq) goto unequal;
     eq = co->co_firstlineno == cp->co_firstlineno;
     if (!eq) goto unequal;
-    eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ);
-    if (eq <= 0) goto unequal;
+    PyObject *co_code = _PyCode_GetCode(co);
+    if (co_code == NULL) {
+        return NULL;
+    }
+    PyObject *cp_code = _PyCode_GetCode(cp);
+    if (cp_code == NULL) {
+        Py_DECREF(co_code);
+        return NULL;
+    }
+    eq = PyObject_RichCompareBool(co_code, cp_code, Py_EQ);
+    Py_DECREF(co_code);
+    Py_DECREF(cp_code);
+    if (eq <= 0) {
+        goto unequal;
+    }
 
     /* compare constants */
     consts1 = _PyCode_ConstantKey(co->co_consts);
@@ -1465,18 +1456,16 @@ code_richcompare(PyObject *self, PyObject *other, int op)
 static Py_hash_t
 code_hash(PyCodeObject *co)
 {
-    Py_hash_t h, h0, h1, h2, h3, h4;
+    Py_hash_t h, h0, h1, h2, h3;
     h0 = PyObject_Hash(co->co_name);
     if (h0 == -1) return -1;
-    h1 = PyObject_Hash(co->co_code);
+    h1 = PyObject_Hash(co->co_consts);
     if (h1 == -1) return -1;
-    h2 = PyObject_Hash(co->co_consts);
+    h2 = PyObject_Hash(co->co_names);
     if (h2 == -1) return -1;
-    h3 = PyObject_Hash(co->co_names);
+    h3 = PyObject_Hash(co->co_localsplusnames);
     if (h3 == -1) return -1;
-    h4 = PyObject_Hash(co->co_localsplusnames);
-    if (h4 == -1) return -1;
-    h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^
+    h = h0 ^ h1 ^ h2 ^ h3 ^
         co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^
         co->co_flags;
     if (h == -1) h = -2;
@@ -1487,22 +1476,22 @@ code_hash(PyCodeObject *co)
 #define OFF(x) offsetof(PyCodeObject, x)
 
 static PyMemberDef code_memberlist[] = {
-    {"co_argcount",     T_INT,          OFF(co_argcount),        READONLY},
-    {"co_posonlyargcount",      T_INT,  OFF(co_posonlyargcount), READONLY},
-    {"co_kwonlyargcount",       T_INT,  OFF(co_kwonlyargcount),  READONLY},
-    {"co_stacksize",T_INT,              OFF(co_stacksize),       READONLY},
-    {"co_flags",        T_INT,          OFF(co_flags),           READONLY},
-    {"co_code",         T_OBJECT,       OFF(co_code),            READONLY},
-    {"co_consts",       T_OBJECT,       OFF(co_consts),          READONLY},
-    {"co_names",        T_OBJECT,       OFF(co_names),           READONLY},
-    {"co_filename",     T_OBJECT,       OFF(co_filename),        READONLY},
-    {"co_name",         T_OBJECT,       OFF(co_name),            READONLY},
-    {"co_qualname",     T_OBJECT,       OFF(co_qualname),        READONLY},
-    {"co_firstlineno",  T_INT,          OFF(co_firstlineno),     READONLY},
-    {"co_linetable",    T_OBJECT,       OFF(co_linetable),       READONLY},
-    {"co_endlinetable", T_OBJECT,       OFF(co_endlinetable),    READONLY},
-    {"co_columntable",  T_OBJECT,       OFF(co_columntable),     READONLY},
-    {"co_exceptiontable",    T_OBJECT,  OFF(co_exceptiontable),  READONLY},
+    {"co_argcount",        T_INT,    OFF(co_argcount),        READONLY},
+    {"co_posonlyargcount", T_INT,    OFF(co_posonlyargcount), READONLY},
+    {"co_kwonlyargcount",  T_INT,    OFF(co_kwonlyargcount),  READONLY},
+    {"co_stacksize",       T_INT,    OFF(co_stacksize),       READONLY},
+    {"co_flags",           T_INT,    OFF(co_flags),           READONLY},
+    {"co_nlocals",         T_INT,    OFF(co_nlocals),         READONLY},
+    {"co_consts",          T_OBJECT, OFF(co_consts),          READONLY},
+    {"co_names",           T_OBJECT, OFF(co_names),           READONLY},
+    {"co_filename",        T_OBJECT, OFF(co_filename),        READONLY},
+    {"co_name",            T_OBJECT, OFF(co_name),            READONLY},
+    {"co_qualname",        T_OBJECT, OFF(co_qualname),        READONLY},
+    {"co_firstlineno",     T_INT,    OFF(co_firstlineno),     READONLY},
+    {"co_linetable",       T_OBJECT, OFF(co_linetable),       READONLY},
+    {"co_endlinetable",    T_OBJECT, OFF(co_endlinetable),    READONLY},
+    {"co_columntable",     T_OBJECT, OFF(co_columntable),     READONLY},
+    {"co_exceptiontable",  T_OBJECT, OFF(co_exceptiontable),  READONLY},
     {NULL}      /* Sentinel */
 };
 
@@ -1514,12 +1503,6 @@ code_getlnotab(PyCodeObject *code, void *closure)
 }
 
 static PyObject *
-code_getnlocals(PyCodeObject *code, void *closure)
-{
-    return PyLong_FromLong(code->co_nlocals);
-}
-
-static PyObject *
 code_getvarnames(PyCodeObject *code, void *closure)
 {
     return _PyCode_GetVarnames(code);
@@ -1538,23 +1521,26 @@ code_getfreevars(PyCodeObject *code, void *closure)
 }
 
 static PyObject *
-code_getquickened(PyCodeObject *code, void *closure)
+code_getcodeadaptive(PyCodeObject *code, void *closure)
 {
-    if (code->co_quickened == NULL) {
-        Py_RETURN_NONE;
-    }
-    return PyBytes_FromStringAndSize((char *)code->co_firstinstr,
-                                     PyBytes_Size(code->co_code));
+    return PyMemoryView_FromMemory(code->co_code_adaptive, _PyCode_NBYTES(code),
+                                   PyBUF_READ);
+}
+
+static PyObject *
+code_getcode(PyCodeObject *code, void *closure)
+{
+    return _PyCode_GetCode(code);
 }
 
 static PyGetSetDef code_getsetlist[] = {
-    {"co_lnotab",    (getter)code_getlnotab, NULL, NULL},
+    {"co_lnotab",         (getter)code_getlnotab,       NULL, NULL},
+    {"_co_code_adaptive", (getter)code_getcodeadaptive, NULL, NULL},
     // The following old names are kept for backward compatibility.
-    {"co_nlocals",   (getter)code_getnlocals, NULL, NULL},
-    {"co_varnames",  (getter)code_getvarnames, NULL, NULL},
-    {"co_cellvars",  (getter)code_getcellvars, NULL, NULL},
-    {"co_freevars",  (getter)code_getfreevars, NULL, NULL},
-    {"_co_quickened",  (getter)code_getquickened, NULL, NULL},
+    {"co_varnames",       (getter)code_getvarnames,     NULL, NULL},
+    {"co_cellvars",       (getter)code_getcellvars,     NULL, NULL},
+    {"co_freevars",       (getter)code_getfreevars,     NULL, NULL},
+    {"co_code",           (getter)code_getcode,         NULL, NULL},
     {0}
 };
 
@@ -1562,7 +1548,7 @@ static PyGetSetDef code_getsetlist[] = {
 static PyObject *
 code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
 {
-    Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co));
+    Py_ssize_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co));
 
     _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra;
     if (co_extra != NULL) {
@@ -1570,10 +1556,6 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
                (co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]);
     }
 
-    if (co->co_quickened != NULL) {
-        res += PyBytes_GET_SIZE(co->co_code);
-    }
-
     return PyLong_FromSsize_t(res);
 }
 
@@ -1594,12 +1576,12 @@ code.replace
     co_stacksize: int(c_default="self->co_stacksize") = -1
     co_flags: int(c_default="self->co_flags") = -1
     co_firstlineno: int(c_default="self->co_firstlineno") = -1
-    co_code: PyBytesObject(c_default="(PyBytesObject *)self->co_code") = None
+    co_code: PyBytesObject(c_default="NULL") = None
     co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None
     co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None
-    co_varnames: object(subclass_of="&PyTuple_Type", c_default="self->co_varnames") = None
-    co_freevars: object(subclass_of="&PyTuple_Type", c_default="self->co_freevars") = None
-    co_cellvars: object(subclass_of="&PyTuple_Type", c_default="self->co_cellvars") = None
+    co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
+    co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
+    co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
     co_filename: unicode(c_default="self->co_filename") = None
     co_name: unicode(c_default="self->co_name") = None
     co_qualname: unicode(c_default="self->co_qualname") = None
@@ -1622,7 +1604,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
                   PyObject *co_name, PyObject *co_qualname,
                   PyBytesObject *co_linetable, PyObject *co_endlinetable,
                   PyObject *co_columntable, PyBytesObject *co_exceptiontable)
-/*[clinic end generated code: output=f046bf0be3bab91f input=a63d09f248f00794]*/
+/*[clinic end generated code: output=f046bf0be3bab91f input=78dbe204dbd06c2f]*/
 {
 #define CHECK_INT_ARG(ARG) \
         if (ARG < 0) { \
@@ -1641,6 +1623,15 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
 
 #undef CHECK_INT_ARG
 
+    PyObject *code = NULL;
+    if (co_code == NULL) {
+        code = _PyCode_GetCode(self);
+        if (code == NULL) {
+            return NULL;
+        }
+        co_code = (PyBytesObject *)code;
+    }
+
     if (PySys_Audit("code.__new__", "OOOiiiiii",
                     co_code, co_filename, co_name, co_argcount,
                     co_posonlyargcount, co_kwonlyargcount, co_nlocals,
@@ -1694,6 +1685,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
         (PyObject*)co_exceptiontable);
 
 error:
+    Py_XDECREF(code);
     Py_XDECREF(varnames);
     Py_XDECREF(cellvars);
     Py_XDECREF(freevars);
@@ -1737,8 +1729,8 @@ static struct PyMethodDef code_methods[] = {
 PyTypeObject PyCode_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "code",
-    sizeof(PyCodeObject),
-    0,
+    offsetof(PyCodeObject, co_code_adaptive),
+    sizeof(_Py_CODEUNIT),
     (destructor)code_dealloc,           /* tp_dealloc */
     0,                                  /* tp_vectorcall_offset */
     0,                                  /* tp_getattr */
@@ -1913,15 +1905,12 @@ _PyCode_ConstantKey(PyObject *op)
 void
 _PyStaticCode_Dealloc(PyCodeObject *co)
 {
-    if (co->co_quickened) {
-        PyMem_Free(co->co_quickened);
-        co->co_quickened = NULL;
+    if (co->co_warmup == 0) {
          _Py_QuickenedCount--;
     }
     co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
     PyMem_Free(co->co_extra);
     co->co_extra = NULL;
-    co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code);
     if (co->co_weakreflist != NULL) {
         PyObject_ClearWeakRefs((PyObject *)co);
         co->co_weakreflist = NULL;
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index 73b6c3d..7ccd300 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -105,8 +105,9 @@ frame_getback(PyFrameObject *f, void *closure)
     return res;
 }
 
-/* Given the index of the effective opcode,
-   scan back to construct the oparg with EXTENDED_ARG */
+// Given the index of the effective opcode, scan back to construct the oparg
+// with EXTENDED_ARG. This only works correctly with *unquickened* code,
+// obtained via a call to _PyCode_GetCode!
 static unsigned int
 get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i)
 {
@@ -170,13 +171,17 @@ top_of_stack(int64_t stack)
 static int64_t *
 mark_stacks(PyCodeObject *code_obj, int len)
 {
-    const _Py_CODEUNIT *code =
-        (const _Py_CODEUNIT *)PyBytes_AS_STRING(code_obj->co_code);
+    PyObject *co_code = _PyCode_GetCode(code_obj);
+    if (co_code == NULL) {
+        return NULL;
+    }
+    _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code);
     int64_t *stacks = PyMem_New(int64_t, len+1);
     int i, j, opcode;
 
     if (stacks == NULL) {
         PyErr_NoMemory();
+        Py_DECREF(co_code);
         return NULL;
     }
     for (int i = 1; i <= len; i++) {
@@ -304,6 +309,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
             }
         }
     }
+    Py_DECREF(co_code);
     return stacks;
 }
 
@@ -493,7 +499,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
 
     /* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this
      * should never overflow. */
-    int len = (int)(PyBytes_GET_SIZE(f->f_frame->f_code->co_code) / sizeof(_Py_CODEUNIT));
+    int len = (int)Py_SIZE(f->f_frame->f_code);
     int *lines = marklines(f->f_frame->f_code, len);
     if (lines == NULL) {
         return -1;
@@ -838,12 +844,23 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
 static int
 _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg)
 {
-    const _Py_CODEUNIT *code =
-        (const _Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code);
+    // This only works when opcode is a non-quickened form:
+    assert(_PyOpcode_Deopt[opcode] == opcode);
+    int check_oparg = 0;
     for (int i = 0; i < frame->f_lasti; i++) {
-        if (_Py_OPCODE(code[i]) == opcode && _Py_OPARG(code[i]) == oparg) {
+        _Py_CODEUNIT instruction = _PyCode_CODE(frame->f_code)[i];
+        int check_opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
+        check_oparg |= _Py_OPARG(instruction);
+        if (check_opcode == opcode && check_oparg == oparg) {
             return 1;
         }
+        if (check_opcode == EXTENDED_ARG) {
+            check_oparg <<= 8;
+        }
+        else {
+            check_oparg = 0;
+        }
+        i += _PyOpcode_Caches[check_opcode];
     }
     return 0;
 }
@@ -862,7 +879,10 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
     }
     co = frame->f_code;
     fast = _PyFrame_GetLocalsArray(frame);
-    if (frame->f_lasti < 0 && _Py_OPCODE(co->co_firstinstr[0]) == COPY_FREE_VARS) {
+    // COPY_FREE_VARS has no quickened forms, so no need to use _PyOpcode_Deopt
+    // here:
+    if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS)
+    {
         /* Free vars have not been initialized -- Do that */
         PyCodeObject *co = frame->f_code;
         PyObject *closure = frame->f_func->func_closure;
@@ -872,6 +892,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
             Py_INCREF(o);
             frame->localsplus[offset + i] = o;
         }
+        // COPY_FREE_VARS doesn't have inline CACHEs, either:
         frame->f_lasti = 0;
     }
     for (int i = 0; i < co->co_nlocalsplus; i++) {
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 6551b93..3ad8dc1 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -349,19 +349,17 @@ _PyGen_yf(PyGenObject *gen)
 
     if (gen->gi_frame_valid) {
         _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe;
-        PyObject *bytecode = gen->gi_code->co_code;
-        unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
 
         if (frame->f_lasti < 1) {
             /* Return immediately if the frame didn't start yet. SEND
                always come after LOAD_CONST: a code object should not start
                with SEND */
-            assert(code[0] != SEND);
+            assert(_Py_OPCODE(_PyCode_CODE(gen->gi_code)[0]) != SEND);
             return NULL;
         }
-        int opcode = code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)];
-        int oparg = code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)+1];
-        if (opcode != RESUME || oparg < 2) {
+        _Py_CODEUNIT next = _PyCode_CODE(gen->gi_code)[frame->f_lasti + 1];
+        if (_PyOpcode_Deopt[_Py_OPCODE(next)] != RESUME || _Py_OPARG(next) < 2)
+        {
             /* Not in a yield from */
             return NULL;
         }
@@ -485,14 +483,15 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
             ret = _PyFrame_StackPop((_PyInterpreterFrame *)gen->gi_iframe);
             assert(ret == yf);
             Py_DECREF(ret);
+            // XXX: Performing this jump ourselves is awkward and problematic.
+            // See https://github.com/python/cpython/pull/31968.
             /* Termination repetition of SEND loop */
             assert(frame->f_lasti >= 0);
-            PyObject *bytecode = gen->gi_code->co_code;
-            unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
+            _Py_CODEUNIT *code = _PyCode_CODE(gen->gi_code);
             /* Backup to SEND */
             frame->f_lasti--;
-            assert(code[frame->f_lasti*sizeof(_Py_CODEUNIT)] == SEND);
-            int jump = code[frame->f_lasti*sizeof(_Py_CODEUNIT)+1];
+            assert(_Py_OPCODE(code[frame->f_lasti]) == SEND);
+            int jump = _Py_OPARG(code[frame->f_lasti]);
             frame->f_lasti += jump;
             if (_PyGen_FetchStopIterationValue(&val) == 0) {
                 ret = gen_send(gen, val);
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 7879515..4bed3ef 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -8949,7 +8949,10 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co,
         // "firstarg" is a cell here unless (very unlikely) super()
         // was called from the C-API before the first MAKE_CELL op.
         if (cframe->f_lasti >= 0) {
-            assert(_Py_OPCODE(*co->co_firstinstr) == MAKE_CELL || _Py_OPCODE(*co->co_firstinstr) == COPY_FREE_VARS);
+            // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need
+            // to use _PyOpcode_Deopt here:
+            assert(_Py_OPCODE(_PyCode_CODE(co)[0]) == MAKE_CELL || 
+                   _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS);
             assert(PyCell_Check(firstarg));
             firstarg = PyCell_GET(firstarg);
         }