bpo-43693: Add new internal code objects fields: co_fastlocalnames and co_fastlocalkinds. (gh-26388)

A number of places in the code base (notably ceval.c and frameobject.c) rely on mapping variable names to indices in the frame "locals plus" array (AKA fast locals), and thus opargs.  Currently the compiler indirectly encodes that information on the code object as the tuples co_varnames, co_cellvars, and co_freevars.  At runtime the dependent code must calculate the proper mapping from those, which isn't ideal and impacts performance-sensitive sections.  This is something we can easily address in the compiler instead.

This change addresses the situation by replacing internal use of co_varnames, etc. with a single combined tuple of names in locals-plus order, along with a minimal array mapping each to its kind (local vs. cell vs. free).  These two new PyCodeObject fields, co_fastlocalnames and co_fastllocalkinds, are not exposed to Python code for now, but co_varnames, etc. are still available with the same values as before (though computed lazily).

Aside from the (mild) performance impact, there are a number of other benefits:

* there's now a clear, direct relationship between locals-plus and variables
* code that relies on the locals-plus-to-name mapping is simpler
* marshaled code objects are smaller and serialize/de-serialize faster

Also note that we can take this approach further by expanding the possible values in co_fastlocalkinds to include specific argument types (e.g. positional-only, kwargs).  Doing so would allow further speed-ups in _PyEval_MakeFrameVector(), which is where args get unpacked into the locals-plus array.  It would also allow us to shrink marshaled code objects even further.

https://bugs.python.org/issue43693

4 files changed, 478 insertions, 221 deletions

diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h
index 7ffdf07..629d265 100644
--- a/Objects/clinic/codeobject.c.h
+++ b/Objects/clinic/codeobject.c.h
@@ -373,4 +373,51 @@ skip_optional_kwonly:
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=a272b22f63ea002e input=a9049054013a1b77]*/
+
+PyDoc_STRVAR(code__varname_from_oparg__doc__,
+"_varname_from_oparg($self, /, oparg, *, cell=False)\n"
+"--\n"
+"\n"
+"(internal-only) Return the local variable name for the given oparg.\n"
+"\n"
+"WARNING: this method is for internal use only and may change or go away.");
+
+#define CODE__VARNAME_FROM_OPARG_METHODDEF    \
+    {"_varname_from_oparg", (PyCFunction)(void(*)(void))code__varname_from_oparg, METH_FASTCALL|METH_KEYWORDS, code__varname_from_oparg__doc__},
+
+static PyObject *
+code__varname_from_oparg_impl(PyCodeObject *self, int oparg, int cell);
+
+static PyObject *
+code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"oparg", "cell", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "_varname_from_oparg", 0};
+    PyObject *argsbuf[2];
+    Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
+    int oparg;
+    int cell = 0;
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    oparg = _PyLong_AsInt(args[0]);
+    if (oparg == -1 && PyErr_Occurred()) {
+        goto exit;
+    }
+    if (!noptargs) {
+        goto skip_optional_kwonly;
+    }
+    cell = PyObject_IsTrue(args[1]);
+    if (cell < 0) {
+        goto exit;
+    }
+skip_optional_kwonly:
+    return_value = code__varname_from_oparg_impl(self, oparg, cell);
+
+exit:
+    return return_value;
+}
+/*[clinic end generated code: output=43f4eef80d584fe0 input=a9049054013a1b77]*/
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index 75e8182..051aefe 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -154,6 +154,93 @@ validate_and_copy_tuple(PyObject *tup)
  * _PyCode_New()
  ******************/
 
+// This is also used in compile.c.
+void
+_Py_set_localsplus_info(int offset, PyObject *name, _PyLocalsPlusKind kind,
+                       PyObject *names, _PyLocalsPlusKinds kinds)
+{
+    Py_INCREF(name);
+    PyTuple_SET_ITEM(names, offset, name);
+    kinds[offset] = kind;
+
+    if (kind == CO_FAST_CELL) {
+        // Cells can overlap with args, so mark those cases.
+        int nlocalsplus = (int)PyTuple_GET_SIZE(names);
+        for (int i = 0; i < nlocalsplus; i++) {
+            _PyLocalsPlusKind kind = kinds[i];
+            if (kind && !(kind & CO_FAST_LOCAL)) {
+                // We've moved past the locals.
+                break;
+            }
+            PyObject *varname = PyTuple_GET_ITEM(names, i);
+            int cmp = PyUnicode_Compare(name, varname);
+            if (cmp == 0) {
+                kinds[i] |= CO_FAST_CELL;
+                break;
+            }
+            assert(cmp > 0 || !PyErr_Occurred());
+        }
+    }
+}
+
+static void
+get_localsplus_counts(PyObject *names, _PyLocalsPlusKinds kinds,
+                      int *pnlocals, int *pncellvars,
+                      int *pnfreevars)
+{
+    int nlocals = 0;
+    int ncellvars = 0;
+    int nfreevars = 0;
+    int nlocalsplus = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(names),
+                                       Py_ssize_t, int);
+    for (int i = 0; i < nlocalsplus; i++) {
+        if (kinds[i] & CO_FAST_LOCAL) {
+            nlocals += 1;
+        }
+        else if (kinds[i] & CO_FAST_CELL) {
+            ncellvars += 1;
+        }
+        else if (kinds[i] & CO_FAST_FREE) {
+            nfreevars += 1;
+        }
+    }
+    if (pnlocals != NULL) {
+        *pnlocals = nlocals;
+    }
+    if (pncellvars != NULL) {
+        *pncellvars = ncellvars;
+    }
+    if (pnfreevars != NULL) {
+        *pnfreevars = nfreevars;
+    }
+}
+
+static PyObject *
+get_localsplus_names(PyCodeObject *co, _PyLocalsPlusKind kind, int num)
+{
+    PyObject *names = PyTuple_New(num);
+    if (names == NULL) {
+        return NULL;
+    }
+    int index = 0;
+    for (int offset = 0; offset < co->co_nlocalsplus; offset++) {
+        if ((co->co_localspluskinds[offset] & kind) == 0) {
+            continue;
+        }
+        // For now there may be duplicates, which we ignore.
+        if (kind == CO_FAST_CELL && co->co_localspluskinds[offset] != kind) {
+            continue;
+        }
+        assert(index < num);
+        PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, offset);
+        Py_INCREF(name);
+        PyTuple_SET_ITEM(names, index, name);
+        index += 1;
+    }
+    assert(index == num);
+    return names;
+}
+
 int
 _PyCode_Validate(struct _PyCodeConstructor *con)
 {
@@ -164,9 +251,9 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
         con->code == NULL || !PyBytes_Check(con->code) ||
         con->consts == NULL || !PyTuple_Check(con->consts) ||
         con->names == NULL || !PyTuple_Check(con->names) ||
-        con->varnames == NULL || !PyTuple_Check(con->varnames) ||
-        con->freevars == NULL || !PyTuple_Check(con->freevars) ||
-        con->cellvars == NULL || !PyTuple_Check(con->cellvars) ||
+        con->localsplusnames == NULL || !PyTuple_Check(con->localsplusnames) ||
+        (PyTuple_GET_SIZE(con->localsplusnames) && con->localspluskinds == NULL) ||
+        (!PyTuple_GET_SIZE(con->localsplusnames) && con->localspluskinds != NULL) ||
         con->name == NULL || !PyUnicode_Check(con->name) ||
         con->filename == NULL || !PyUnicode_Check(con->filename) ||
         con->linetable == NULL || !PyBytes_Check(con->linetable) ||
@@ -187,7 +274,10 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
     /* Ensure that the co_varnames has enough names to cover the arg counts.
      * Note that totalargs = nlocals - nplainlocals.  We check nplainlocals
      * here to avoid the possibility of overflow (however remote). */
-    int nplainlocals = (int)PyTuple_GET_SIZE(con->varnames) -
+    int nlocals;
+    get_localsplus_counts(con->localsplusnames, con->localspluskinds,
+                          &nlocals, NULL, NULL);
+    int nplainlocals = nlocals -
                        con->argcount -
                        con->kwonlyargcount -
                        ((con->flags & CO_VARARGS) != 0) -
@@ -203,6 +293,11 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
 static void
 init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
 {
+    int nlocalsplus = (int)PyTuple_GET_SIZE(con->localsplusnames);
+    int nlocals, ncellvars, nfreevars;
+    get_localsplus_counts(con->localsplusnames, con->localspluskinds,
+                          &nlocals, &ncellvars, &nfreevars);
+
     Py_INCREF(con->filename);
     co->co_filename = con->filename;
     Py_INCREF(con->name);
@@ -220,12 +315,10 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
     Py_INCREF(con->names);
     co->co_names = con->names;
 
-    Py_INCREF(con->varnames);
-    co->co_varnames = con->varnames;
-    Py_INCREF(con->cellvars);
-    co->co_cellvars = con->cellvars;
-    Py_INCREF(con->freevars);
-    co->co_freevars = con->freevars;
+    Py_INCREF(con->localsplusnames);
+    co->co_localsplusnames = con->localsplusnames;
+    // We take ownership of the kinds array.
+    co->co_localspluskinds = con->localspluskinds;
 
     co->co_argcount = con->argcount;
     co->co_posonlyargcount = con->posonlyargcount;
@@ -238,10 +331,13 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
 
     /* derived values */
     co->co_cell2arg = NULL;  // This will be set soon.
-    co->co_nlocals = (int)PyTuple_GET_SIZE(con->varnames);
-    co->co_ncellvars = (int)PyTuple_GET_SIZE(con->cellvars);
-    co->co_nfreevars = (int)PyTuple_GET_SIZE(con->freevars);
-    co->co_nlocalsplus = co->co_nlocals + co->co_ncellvars + co->co_nfreevars;
+    co->co_nlocalsplus = nlocalsplus;
+    co->co_nlocals = nlocals;
+    co->co_ncellvars = ncellvars;
+    co->co_nfreevars = nfreevars;
+    co->co_varnames = NULL;
+    co->co_cellvars = NULL;
+    co->co_freevars = NULL;
 
     /* not set */
     co->co_weakreflist = NULL;
@@ -271,23 +367,9 @@ _PyCode_New(struct _PyCodeConstructor *con)
     if (intern_string_constants(con->consts, NULL) < 0) {
         return NULL;
     }
-    if (intern_strings(con->varnames) < 0) {
+    if (intern_strings(con->localsplusnames) < 0) {
         return NULL;
     }
-    if (intern_strings(con->freevars) < 0) {
-        return NULL;
-    }
-    if (intern_strings(con->cellvars) < 0) {
-        return NULL;
-    }
-
-    /* Check for any inner or outer closure references */
-    int ncellvars = (int)PyTuple_GET_SIZE(con->cellvars);
-    if (!ncellvars && !PyTuple_GET_SIZE(con->freevars)) {
-        con->flags |= CO_NOFREE;
-    } else {
-        con->flags &= ~CO_NOFREE;
-    }
 
     PyCodeObject *co = PyObject_New(PyCodeObject, &PyCode_Type);
     if (co == NULL) {
@@ -296,18 +378,26 @@ _PyCode_New(struct _PyCodeConstructor *con)
     }
     init_code(co, con);
 
+    /* Check for any inner or outer closure references */
+    if (!co->co_ncellvars && !co->co_nfreevars) {
+        co->co_flags |= CO_NOFREE;
+    } else {
+        co->co_flags &= ~CO_NOFREE;
+    }
+
     /* Create mapping between cells and arguments if needed. */
-    if (ncellvars) {
+    if (co->co_ncellvars) {
         int totalargs = co->co_argcount +
                         co->co_kwonlyargcount +
                         ((co->co_flags & CO_VARARGS) != 0) +
                         ((co->co_flags & CO_VARKEYWORDS) != 0);
         assert(totalargs <= co->co_nlocals);
         /* Find cells which are also arguments. */
-        for (int i = 0; i < ncellvars; i++) {
-            PyObject *cellname = PyTuple_GET_ITEM(co->co_cellvars, i);
+        for (int i = 0; i < co->co_ncellvars; i++) {
+            PyObject *cellname = PyTuple_GET_ITEM(co->co_localsplusnames,
+                                                  i + co->co_nlocals);
             for (int j = 0; j < totalargs; j++) {
-                PyObject *argname = PyTuple_GET_ITEM(co->co_varnames, j);
+                PyObject *argname = PyTuple_GET_ITEM(co->co_localsplusnames, j);
                 int cmp = PyUnicode_Compare(cellname, argname);
                 if (cmp == -1 && PyErr_Occurred()) {
                     Py_DECREF(co);
@@ -315,13 +405,13 @@ _PyCode_New(struct _PyCodeConstructor *con)
                 }
                 if (cmp == 0) {
                     if (co->co_cell2arg == NULL) {
-                        co->co_cell2arg = PyMem_NEW(int, ncellvars);
+                        co->co_cell2arg = PyMem_NEW(int, co->co_ncellvars);
                         if (co->co_cell2arg == NULL) {
                             Py_DECREF(co);
                             PyErr_NoMemory();
                             return NULL;
                         }
-                        for (int k = 0; k < ncellvars; k++) {
+                        for (int k = 0; k < co->co_ncellvars; k++) {
                             co->co_cell2arg[k] = CO_CELL_NOT_AN_ARG;
                         }
                     }
@@ -349,6 +439,47 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
                           PyObject *filename, PyObject *name, int firstlineno,
                           PyObject *linetable, PyObject *exceptiontable)
 {
+    PyCodeObject *co = NULL;
+    PyObject *localsplusnames = NULL;
+    _PyLocalsPlusKinds localspluskinds = NULL;
+
+    if (varnames == NULL || !PyTuple_Check(varnames) ||
+        cellvars == NULL || !PyTuple_Check(cellvars) ||
+        freevars == NULL || !PyTuple_Check(freevars)
+        ) {
+        PyErr_BadInternalCall();
+        return NULL;
+    }
+
+    // Set the "fast locals plus" info.
+    int nvarnames = (int)PyTuple_GET_SIZE(varnames);
+    int ncellvars = (int)PyTuple_GET_SIZE(cellvars);
+    int nfreevars = (int)PyTuple_GET_SIZE(freevars);
+    int nlocalsplus = nvarnames + ncellvars + nfreevars;
+    localsplusnames = PyTuple_New(nlocalsplus);
+    if (localsplusnames == NULL) {
+        goto error;
+    }
+    if (_PyCode_InitLocalsPlusKinds(nlocalsplus, &localspluskinds) < 0) {
+        goto error;
+    }
+    int  offset = 0;
+    for (int i = 0; i < nvarnames; i++, offset++) {
+        PyObject *name = PyTuple_GET_ITEM(varnames, i);
+        _Py_set_localsplus_info(offset, name, CO_FAST_LOCAL,
+                               localsplusnames, localspluskinds);
+    }
+    for (int i = 0; i < ncellvars; i++, offset++) {
+        PyObject *name = PyTuple_GET_ITEM(cellvars, i);
+        _Py_set_localsplus_info(offset, name, CO_FAST_CELL,
+                               localsplusnames, localspluskinds);
+    }
+    for (int i = 0; i < nfreevars; i++, offset++) {
+        PyObject *name = PyTuple_GET_ITEM(freevars, i);
+        _Py_set_localsplus_info(offset, name, CO_FAST_FREE,
+                               localsplusnames, localspluskinds);
+    }
+
     struct _PyCodeConstructor con = {
         .filename = filename,
         .name = name,
@@ -361,9 +492,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
         .consts = consts,
         .names = names,
 
-        .varnames = varnames,
-        .cellvars = cellvars,
-        .freevars = freevars,
+        .localsplusnames = localsplusnames,
+        .localspluskinds = localspluskinds,
 
         .argcount = argcount,
         .posonlyargcount = posonlyargcount,
@@ -373,17 +503,33 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
 
         .exceptiontable = exceptiontable,
     };
+
     if (_PyCode_Validate(&con) < 0) {
-        return NULL;
+        goto error;
     }
-
     if (nlocals != PyTuple_GET_SIZE(varnames)) {
         PyErr_SetString(PyExc_ValueError,
                         "code: co_nlocals != len(co_varnames)");
-        return NULL;
+        goto error;
+    }
+
+    co = _PyCode_New(&con);
+    if (co == NULL) {
+        goto error;
     }
 
-    return _PyCode_New(&con);
+    localspluskinds = NULL;  // This keeps it from getting freed below.
+    Py_INCREF(varnames);
+    co->co_varnames = varnames;
+    Py_INCREF(cellvars);
+    co->co_cellvars = cellvars;
+    Py_INCREF(freevars);
+    co->co_freevars = freevars;
+
+error:
+    Py_XDECREF(localsplusnames);
+    _PyCode_ClearLocalsPlusKinds(localspluskinds);
+    return co;
 }
 
 PyCodeObject *
@@ -434,9 +580,7 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
         .linetable = emptystring,
         .consts = nulltuple,
         .names = nulltuple,
-        .varnames = nulltuple,
-        .cellvars = nulltuple,
-        .freevars = nulltuple,
+        .localsplusnames = nulltuple,
         .exceptiontable = emptystring,
     };
     result = _PyCode_New(&con);
@@ -880,6 +1024,53 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
 
 
 /******************
+ * other PyCodeObject accessor functions
+ ******************/
+
+PyObject *
+_PyCode_GetVarnames(PyCodeObject *co)
+{
+    if (co->co_varnames == NULL) {
+        co->co_varnames = get_localsplus_names(co, CO_FAST_LOCAL,
+                                               co->co_nlocals);
+        if (co->co_varnames == NULL) {
+            return NULL;
+        }
+    }
+    Py_INCREF(co->co_varnames);
+    return co->co_varnames;
+}
+
+PyObject *
+_PyCode_GetCellvars(PyCodeObject *co)
+{
+    if (co->co_cellvars == NULL) {
+        co->co_cellvars = get_localsplus_names(co, CO_FAST_CELL,
+                                               co->co_ncellvars);
+        if (co->co_cellvars == NULL) {
+            return NULL;
+        }
+    }
+    Py_INCREF(co->co_cellvars);
+    return co->co_cellvars;
+}
+
+PyObject *
+_PyCode_GetFreevars(PyCodeObject *co)
+{
+    if (co->co_freevars == NULL) {
+        co->co_freevars = get_localsplus_names(co, CO_FAST_FREE,
+                                               co->co_nfreevars);
+        if (co->co_freevars == NULL) {
+            return NULL;
+        }
+    }
+    Py_INCREF(co->co_freevars);
+    return co->co_freevars;
+}
+
+
+/******************
  * PyCode_Type
  ******************/
 
@@ -1028,6 +1219,8 @@ code_dealloc(PyCodeObject *co)
     Py_XDECREF(co->co_code);
     Py_XDECREF(co->co_consts);
     Py_XDECREF(co->co_names);
+    Py_XDECREF(co->co_localsplusnames);
+    _PyCode_ClearLocalsPlusKinds(co->co_localspluskinds);
     Py_XDECREF(co->co_varnames);
     Py_XDECREF(co->co_freevars);
     Py_XDECREF(co->co_cellvars);
@@ -1086,8 +1279,6 @@ code_richcompare(PyObject *self, PyObject *other, int op)
     if (!eq) goto unequal;
     eq = co->co_kwonlyargcount == cp->co_kwonlyargcount;
     if (!eq) goto unequal;
-    eq = co->co_nlocals == cp->co_nlocals;
-    if (!eq) goto unequal;
     eq = co->co_flags == cp->co_flags;
     if (!eq) goto unequal;
     eq = co->co_firstlineno == cp->co_firstlineno;
@@ -1111,11 +1302,8 @@ code_richcompare(PyObject *self, PyObject *other, int op)
 
     eq = PyObject_RichCompareBool(co->co_names, cp->co_names, Py_EQ);
     if (eq <= 0) goto unequal;
-    eq = PyObject_RichCompareBool(co->co_varnames, cp->co_varnames, Py_EQ);
-    if (eq <= 0) goto unequal;
-    eq = PyObject_RichCompareBool(co->co_freevars, cp->co_freevars, Py_EQ);
-    if (eq <= 0) goto unequal;
-    eq = PyObject_RichCompareBool(co->co_cellvars, cp->co_cellvars, Py_EQ);
+    eq = PyObject_RichCompareBool(co->co_localsplusnames,
+                                  cp->co_localsplusnames, Py_EQ);
     if (eq <= 0) goto unequal;
 
     if (op == Py_EQ)
@@ -1140,7 +1328,7 @@ code_richcompare(PyObject *self, PyObject *other, int op)
 static Py_hash_t
 code_hash(PyCodeObject *co)
 {
-    Py_hash_t h, h0, h1, h2, h3, h4, h5, h6;
+    Py_hash_t h, h0, h1, h2, h3, h4;
     h0 = PyObject_Hash(co->co_name);
     if (h0 == -1) return -1;
     h1 = PyObject_Hash(co->co_code);
@@ -1149,15 +1337,11 @@ code_hash(PyCodeObject *co)
     if (h2 == -1) return -1;
     h3 = PyObject_Hash(co->co_names);
     if (h3 == -1) return -1;
-    h4 = PyObject_Hash(co->co_varnames);
+    h4 = PyObject_Hash(co->co_localsplusnames);
     if (h4 == -1) return -1;
-    h5 = PyObject_Hash(co->co_freevars);
-    if (h5 == -1) return -1;
-    h6 = PyObject_Hash(co->co_cellvars);
-    if (h6 == -1) return -1;
-    h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6 ^
+    h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^
         co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^
-        co->co_nlocals ^ co->co_flags;
+        co->co_flags;
     if (h == -1) h = -2;
     return h;
 }
@@ -1169,15 +1353,11 @@ static PyMemberDef code_memberlist[] = {
     {"co_argcount",     T_INT,          OFF(co_argcount),        READONLY},
     {"co_posonlyargcount",      T_INT,  OFF(co_posonlyargcount), READONLY},
     {"co_kwonlyargcount",       T_INT,  OFF(co_kwonlyargcount),  READONLY},
-    {"co_nlocals",      T_INT,          OFF(co_nlocals),         READONLY},
     {"co_stacksize",T_INT,              OFF(co_stacksize),       READONLY},
     {"co_flags",        T_INT,          OFF(co_flags),           READONLY},
     {"co_code",         T_OBJECT,       OFF(co_code),            READONLY},
     {"co_consts",       T_OBJECT,       OFF(co_consts),          READONLY},
     {"co_names",        T_OBJECT,       OFF(co_names),           READONLY},
-    {"co_varnames",     T_OBJECT,       OFF(co_varnames),        READONLY},
-    {"co_freevars",     T_OBJECT,       OFF(co_freevars),        READONLY},
-    {"co_cellvars",     T_OBJECT,       OFF(co_cellvars),        READONLY},
     {"co_filename",     T_OBJECT,       OFF(co_filename),        READONLY},
     {"co_name",         T_OBJECT,       OFF(co_name),            READONLY},
     {"co_firstlineno",  T_INT,          OFF(co_firstlineno),     READONLY},
@@ -1187,15 +1367,43 @@ static PyMemberDef code_memberlist[] = {
 };
 
 
-
 static PyObject *
 code_getlnotab(PyCodeObject *code, void *closure)
 {
     return decode_linetable(code);
 }
 
+static PyObject *
+code_getnlocals(PyCodeObject *code, void *closure)
+{
+    return PyLong_FromLong(code->co_nlocals);
+}
+
+static PyObject *
+code_getvarnames(PyCodeObject *code, void *closure)
+{
+    return _PyCode_GetVarnames(code);
+}
+
+static PyObject *
+code_getcellvars(PyCodeObject *code, void *closure)
+{
+    return _PyCode_GetCellvars(code);
+}
+
+static PyObject *
+code_getfreevars(PyCodeObject *code, void *closure)
+{
+    return _PyCode_GetFreevars(code);
+}
+
 static PyGetSetDef code_getsetlist[] = {
     {"co_lnotab",    (getter)code_getlnotab, NULL, NULL},
+    // The following old names are kept for backward compatibility.
+    {"co_nlocals",   (getter)code_getnlocals, NULL, NULL},
+    {"co_varnames",  (getter)code_getvarnames, NULL, NULL},
+    {"co_cellvars",  (getter)code_getcellvars, NULL, NULL},
+    {"co_freevars",  (getter)code_getfreevars, NULL, NULL},
     {0}
 };
 
@@ -1204,15 +1412,17 @@ static PyObject *
 code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
 {
     Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co));
-    _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra;
 
-    if (co->co_cell2arg != NULL && co->co_cellvars != NULL) {
-        res += co->co_ncellvars * sizeof(Py_ssize_t);
-    }
+    _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra;
     if (co_extra != NULL) {
         res += sizeof(_PyCodeObjectExtra) +
                (co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]);
     }
+
+    if (co->co_cell2arg != NULL && co->co_cellvars != NULL) {
+        res += co->co_ncellvars * sizeof(Py_ssize_t);
+    }
+
     if (co->co_opcache != NULL) {
         assert(co->co_opcache_map != NULL);
         // co_opcache_map
@@ -1220,6 +1430,7 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
         // co_opcache
         res += co->co_opcache_size * sizeof(_PyOpcache);
     }
+
     return PyLong_FromSsize_t(res);
 }
 
@@ -1290,11 +1501,73 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
         return NULL;
     }
 
-    return (PyObject *)PyCode_NewWithPosOnlyArgs(
+    PyObject *varnames = NULL;
+    PyObject *cellvars = NULL;
+    PyObject *freevars = NULL;
+    if (co_varnames == NULL) {
+        varnames = get_localsplus_names(self, CO_FAST_LOCAL, self->co_nlocals);
+        if (varnames == NULL) {
+            goto error;
+        }
+        co_varnames = varnames;
+    }
+    if (co_cellvars == NULL) {
+        cellvars = get_localsplus_names(self, CO_FAST_CELL, self->co_ncellvars);
+        if (cellvars == NULL) {
+            goto error;
+        }
+        co_cellvars = cellvars;
+    }
+    if (co_freevars == NULL) {
+        freevars = get_localsplus_names(self, CO_FAST_FREE, self->co_nfreevars);
+        if (freevars == NULL) {
+            goto error;
+        }
+        co_freevars = freevars;
+    }
+
+    PyCodeObject *co = PyCode_NewWithPosOnlyArgs(
         co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals,
         co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names,
         co_varnames, co_freevars, co_cellvars, co_filename, co_name,
         co_firstlineno, (PyObject*)co_linetable, (PyObject*)co_exceptiontable);
+    if (co == NULL) {
+        goto error;
+    }
+    return (PyObject *)co;
+
+error:
+    Py_XDECREF(varnames);
+    Py_XDECREF(cellvars);
+    Py_XDECREF(freevars);
+    return NULL;
+}
+
+/*[clinic input]
+code._varname_from_oparg
+
+    oparg: int
+    *
+    cell: bool = False
+
+(internal-only) Return the local variable name for the given oparg.
+
+WARNING: this method is for internal use only and may change or go away.
+[clinic start generated code]*/
+
+static PyObject *
+code__varname_from_oparg_impl(PyCodeObject *self, int oparg, int cell)
+/*[clinic end generated code: output=c7d39c9723692c8f input=2945bb291d3a3118]*/
+{
+    if (cell) {
+        oparg += self->co_nlocals;
+    }
+    PyObject *name = PyTuple_GetItem(self->co_localsplusnames, oparg);
+    if (name == NULL) {
+        return NULL;
+    }
+    Py_INCREF(name);
+    return name;
 }
 
 /* XXX code objects need to participate in GC? */
@@ -1303,6 +1576,7 @@ static struct PyMethodDef code_methods[] = {
     {"__sizeof__", (PyCFunction)code_sizeof, METH_NOARGS},
     {"co_lines", (PyCFunction)code_linesiterator, METH_NOARGS},
     CODE_REPLACE_METHODDEF
+    CODE__VARNAME_FROM_OPARG_METHODDEF
     {NULL, NULL}                /* sentinel */
 };
 
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index 1bfae90..ef5ff4e 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -4,6 +4,7 @@
 #include "pycore_ceval.h"         // _PyEval_BuiltinsFromGlobals()
 #include "pycore_moduleobject.h"  // _PyModule_GetDict()
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
+#include "pycore_code.h"          // CO_FAST_LOCAL, etc.
 
 #include "frameobject.h"          // PyFrameObject
 #include "pycore_frame.h"
@@ -659,9 +660,9 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
     Py_VISIT(f->f_trace);
 
     /* locals */
-    PyObject **fastlocals = f->f_localsptr;
-    for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++fastlocals) {
-        Py_VISIT(*fastlocals);
+    PyObject **localsplus = f->f_localsptr;
+    for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++localsplus) {
+        Py_VISIT(*localsplus);
     }
 
     /* stack */
@@ -684,9 +685,9 @@ frame_tp_clear(PyFrameObject *f)
     Py_CLEAR(f->f_trace);
 
     /* locals */
-    PyObject **fastlocals = f->f_localsptr;
-    for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++fastlocals) {
-        Py_CLEAR(*fastlocals);
+    PyObject **localsplus = f->f_localsptr;
+    for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++localsplus) {
+        Py_CLEAR(*localsplus);
     }
 
     /* stack */
@@ -917,112 +918,13 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
     return f;
 }
 
-/* Convert between "fast" version of locals and dictionary version.
-
-   map and values are input arguments.  map is a tuple of strings.
-   values is an array of PyObject*.  At index i, map[i] is the name of
-   the variable with value values[i].  The function copies the first
-   nmap variable from map/values into dict.  If values[i] is NULL,
-   the variable is deleted from dict.
-
-   If deref is true, then the values being copied are cell variables
-   and the value is extracted from the cell variable before being put
-   in dict.
- */
-
-static int
-map_to_dict(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values,
-            int deref)
-{
-    Py_ssize_t j;
-    assert(PyTuple_Check(map));
-    assert(PyDict_Check(dict));
-    assert(PyTuple_Size(map) >= nmap);
-    for (j=0; j < nmap; j++) {
-        PyObject *key = PyTuple_GET_ITEM(map, j);
-        PyObject *value = values[j];
-        assert(PyUnicode_Check(key));
-        if (deref && value != NULL) {
-            assert(PyCell_Check(value));
-            value = PyCell_GET(value);
-        }
-        if (value == NULL) {
-            if (PyObject_DelItem(dict, key) != 0) {
-                if (PyErr_ExceptionMatches(PyExc_KeyError))
-                    PyErr_Clear();
-                else
-                    return -1;
-            }
-        }
-        else {
-            if (PyObject_SetItem(dict, key, value) != 0)
-                return -1;
-        }
-    }
-    return 0;
-}
-
-/* Copy values from the "locals" dict into the fast locals.
-
-   dict is an input argument containing string keys representing
-   variables names and arbitrary PyObject* as values.
-
-   map and values are input arguments.  map is a tuple of strings.
-   values is an array of PyObject*.  At index i, map[i] is the name of
-   the variable with value values[i].  The function copies the first
-   nmap variable from map/values into dict.  If values[i] is NULL,
-   the variable is deleted from dict.
-
-   If deref is true, then the values being copied are cell variables
-   and the value is extracted from the cell variable before being put
-   in dict.  If clear is true, then variables in map but not in dict
-   are set to NULL in map; if clear is false, variables missing in
-   dict are ignored.
-
-   Exceptions raised while modifying the dict are silently ignored,
-   because there is no good way to report them.
-*/
-
-static void
-dict_to_map(PyObject *map, Py_ssize_t nmap, PyObject *dict, PyObject **values,
-            int deref, int clear)
-{
-    Py_ssize_t j;
-    assert(PyTuple_Check(map));
-    assert(PyDict_Check(dict));
-    assert(PyTuple_Size(map) >= nmap);
-    for (j=0; j < nmap; j++) {
-        PyObject *key = PyTuple_GET_ITEM(map, j);
-        PyObject *value = PyObject_GetItem(dict, key);
-        assert(PyUnicode_Check(key));
-        /* We only care about NULLs if clear is true. */
-        if (value == NULL) {
-            PyErr_Clear();
-            if (!clear)
-                continue;
-        }
-        if (deref) {
-            assert(PyCell_Check(values[j]));
-            if (PyCell_GET(values[j]) != value) {
-                if (PyCell_Set(values[j], value) < 0)
-                    PyErr_Clear();
-            }
-        } else if (values[j] != value) {
-            Py_XINCREF(value);
-            Py_XSETREF(values[j], value);
-        }
-        Py_XDECREF(value);
-    }
-}
-
 int
 PyFrame_FastToLocalsWithError(PyFrameObject *f)
 {
     /* Merge fast locals into f->f_locals */
-    PyObject *locals, *map;
+    PyObject *locals;
     PyObject **fast;
     PyCodeObject *co;
-    Py_ssize_t j;
 
     if (f == NULL) {
         PyErr_BadInternalCall();
@@ -1035,25 +937,9 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
             return -1;
     }
     co = f->f_code;
-    map = co->co_varnames;
-    if (!PyTuple_Check(map)) {
-        PyErr_Format(PyExc_SystemError,
-                     "co_varnames must be a tuple, not %s",
-                     Py_TYPE(map)->tp_name);
-        return -1;
-    }
     fast = f->f_localsptr;
-    j = PyTuple_GET_SIZE(map);
-    if (j > co->co_nlocals)
-        j = co->co_nlocals;
-    if (co->co_nlocals) {
-        if (map_to_dict(map, j, locals, fast, 0) < 0)
-            return -1;
-    }
-    if (co->co_ncellvars || co->co_nfreevars) {
-        if (map_to_dict(co->co_cellvars, co->co_ncellvars,
-                        locals, fast + co->co_nlocals, 1))
-            return -1;
+    for (int i = 0; i < co->co_nlocalsplus; i++) {
+        _PyLocalsPlusKind kind = co->co_localspluskinds[i];
 
         /* If the namespace is unoptimized, then one of the
            following cases applies:
@@ -1063,10 +949,42 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
            We don't want to accidentally copy free variables
            into the locals dict used by the class.
         */
-        if (co->co_flags & CO_OPTIMIZED) {
-            if (map_to_dict(co->co_freevars, co->co_nfreevars, locals,
-                            fast + co->co_nlocals + co->co_ncellvars, 1) < 0)
+        if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) {
+            continue;
+        }
+
+        /* Some args are also cells.  For now each of those variables
+           has two indices in the fast array, with both marked as cells
+           but only one marked as an arg.  That one is always set
+           to NULL in _PyEval_MakeFrameVector() and the other index
+           gets the cell holding the arg value.  So we ignore the
+           former here and will later use the cell for the variable.
+        */
+        if (kind & CO_FAST_LOCAL && kind & CO_FAST_CELL) {
+            assert(fast[i] == NULL);
+            continue;
+        }
+
+        PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
+        PyObject *value = fast[i];
+        if (kind & (CO_FAST_CELL | CO_FAST_FREE) && value != NULL) {
+            assert(PyCell_Check(value));
+            value = PyCell_GET(value);
+        }
+        if (value == NULL) {
+            if (PyObject_DelItem(locals, name) != 0) {
+                if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+                    PyErr_Clear();
+                }
+                else {
+                    return -1;
+                }
+            }
+        }
+        else {
+            if (PyObject_SetItem(locals, name, value) != 0) {
                 return -1;
+            }
         }
     }
     return 0;
@@ -1088,36 +1006,51 @@ void
 PyFrame_LocalsToFast(PyFrameObject *f, int clear)
 {
     /* Merge locals into fast locals */
-    PyObject *locals, *map;
+    PyObject *locals;
     PyObject **fast;
     PyObject *error_type, *error_value, *error_traceback;
     PyCodeObject *co;
-    Py_ssize_t j;
     if (f == NULL)
         return;
     locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET];
-    co = f->f_code;
-    map = co->co_varnames;
     if (locals == NULL)
         return;
-    if (!PyTuple_Check(map))
-        return;
-    PyErr_Fetch(&error_type, &error_value, &error_traceback);
     fast = f->f_localsptr;
-    j = PyTuple_GET_SIZE(map);
-    if (j > co->co_nlocals)
-        j = co->co_nlocals;
-    if (co->co_nlocals)
-        dict_to_map(co->co_varnames, j, locals, fast, 0, clear);
-    if (co->co_ncellvars || co->co_nfreevars) {
-        dict_to_map(co->co_cellvars, co->co_ncellvars,
-                    locals, fast + co->co_nlocals, 1, clear);
+    co = f->f_code;
+
+    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+    for (int i = 0; i < co->co_nlocalsplus; i++) {
+        _PyLocalsPlusKind kind = co->co_localspluskinds[i];
+
+        /* Same test as in PyFrame_FastToLocals() above. */
+        if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) {
+            continue;
+        }
         /* Same test as in PyFrame_FastToLocals() above. */
-        if (co->co_flags & CO_OPTIMIZED) {
-            dict_to_map(co->co_freevars, co->co_nfreevars, locals,
-                        fast + co->co_nlocals + co->co_ncellvars, 1,
-                        clear);
+        if (kind & CO_FAST_LOCAL && kind & CO_FAST_CELL) {
+            continue;
         }
+        PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
+        PyObject *value = PyObject_GetItem(locals, name);
+        /* We only care about NULLs if clear is true. */
+        if (value == NULL) {
+            PyErr_Clear();
+            if (!clear) {
+                continue;
+            }
+        }
+        if (kind & (CO_FAST_CELL | CO_FAST_FREE)) {
+            assert(PyCell_Check(fast[i]));
+            if (PyCell_GET(fast[i]) != value) {
+                if (PyCell_Set(fast[i], value) < 0) {
+                    PyErr_Clear();
+                }
+            }
+        } else if (fast[i] != value) {
+            Py_XINCREF(value);
+            Py_XSETREF(fast[i], value);
+        }
+        Py_XDECREF(value);
     }
     PyErr_Restore(error_type, error_value, error_traceback);
 }
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index bf792e2..bd2cade 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -2,6 +2,7 @@
 
 #include "Python.h"
 #include "pycore_call.h"
+#include "pycore_code.h"          // CO_FAST_FREE
 #include "pycore_compile.h"       // _Py_Mangle()
 #include "pycore_initconfig.h"
 #include "pycore_moduleobject.h"  // _PyModule_GetDef()
@@ -8894,13 +8895,15 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co,
         return -1;
     }
 
+    // Look for __class__ in the free vars.
     PyTypeObject *type = NULL;
-    for (i = 0; i < co->co_nfreevars; i++) {
-        PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i);
+    i = co->co_nlocals + co->co_ncellvars;
+    for (; i < co->co_nlocalsplus; i++) {
+        assert(co->co_localspluskinds[i] & CO_FAST_FREE);
+        PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
         assert(PyUnicode_Check(name));
         if (_PyUnicode_EqualToASCIIId(name, &PyId___class__)) {
-            Py_ssize_t index = co->co_nlocals + co->co_ncellvars + i;
-            PyObject *cell = f->f_localsptr[index];
+            PyObject *cell = f->f_localsptr[i];
             if (cell == NULL || !PyCell_Check(cell)) {
                 PyErr_SetString(PyExc_RuntimeError,
                   "super(): bad __class__ cell");