bpo-43950: Add code.co_positions (PEP 657) (GH-26955)

This PR is part of PEP 657 and augments the compiler to emit ending
line numbers as well as starting and ending columns from the AST
into compiled code objects. This allows bytecodes to be correlated
to the exact source code ranges that generated them.

This information is made available through the following public APIs:

* The `co_positions` method on code objects.
* The C API function `PyCode_Addr2Location`.

Co-authored-by: Batuhan Taskaya <isidentical@gmail.com>
Co-authored-by: Ammar Askar <ammar@ammaraskar.com>

1 files changed, 60 insertions, 22 deletions

diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h
index c1ad09c..48aa8d7 100644
--- a/Objects/clinic/codeobject.c.h
+++ b/Objects/clinic/codeobject.c.h
@@ -5,8 +5,8 @@ preserve
 PyDoc_STRVAR(code_new__doc__,
 "code(argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize,\n"
 "     flags, codestring, constants, names, varnames, filename, name,\n"
-"     firstlineno, linetable, exceptiontable, freevars=(), cellvars=(),\n"
-"     /)\n"
+"     firstlineno, linetable, endlinetable, columntable, exceptiontable,\n"
+"     freevars=(), cellvars=(), /)\n"
 "--\n"
 "\n"
 "Create a code object.  Not for the faint of heart.");
@@ -16,7 +16,8 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount,
               int kwonlyargcount, int nlocals, int stacksize, int flags,
               PyObject *code, PyObject *consts, PyObject *names,
               PyObject *varnames, PyObject *filename, PyObject *name,
-              int firstlineno, PyObject *linetable, PyObject *exceptiontable,
+              int firstlineno, PyObject *linetable, PyObject *endlinetable,
+              PyObject *columntable, PyObject *exceptiontable,
               PyObject *freevars, PyObject *cellvars);
 
 static PyObject *
@@ -37,6 +38,8 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
     PyObject *name;
     int firstlineno;
     PyObject *linetable;
+    PyObject *endlinetable;
+    PyObject *columntable;
     PyObject *exceptiontable;
     PyObject *freevars = NULL;
     PyObject *cellvars = NULL;
@@ -45,7 +48,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
         !_PyArg_NoKeywords("code", kwargs)) {
         goto exit;
     }
-    if (!_PyArg_CheckPositional("code", PyTuple_GET_SIZE(args), 15, 17)) {
+    if (!_PyArg_CheckPositional("code", PyTuple_GET_SIZE(args), 17, 19)) {
         goto exit;
     }
     argcount = _PyLong_AsInt(PyTuple_GET_ITEM(args, 0));
@@ -121,25 +124,35 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
         _PyArg_BadArgument("code", "argument 15", "bytes", PyTuple_GET_ITEM(args, 14));
         goto exit;
     }
-    exceptiontable = PyTuple_GET_ITEM(args, 14);
-    if (PyTuple_GET_SIZE(args) < 16) {
+    endlinetable = PyTuple_GET_ITEM(args, 14);
+    if (!PyBytes_Check(PyTuple_GET_ITEM(args, 15))) {
+        _PyArg_BadArgument("code", "argument 16", "bytes", PyTuple_GET_ITEM(args, 15));
+        goto exit;
+    }
+    columntable = PyTuple_GET_ITEM(args, 15);
+    if (!PyBytes_Check(PyTuple_GET_ITEM(args, 16))) {
+        _PyArg_BadArgument("code", "argument 17", "bytes", PyTuple_GET_ITEM(args, 16));
+        goto exit;
+    }
+    exceptiontable = PyTuple_GET_ITEM(args, 16);
+    if (PyTuple_GET_SIZE(args) < 18) {
         goto skip_optional;
     }
-    if (!PyTuple_Check(PyTuple_GET_ITEM(args, 15))) {
-        _PyArg_BadArgument("code", "argument 16", "tuple", PyTuple_GET_ITEM(args, 15));
+    if (!PyTuple_Check(PyTuple_GET_ITEM(args, 17))) {
+        _PyArg_BadArgument("code", "argument 18", "tuple", PyTuple_GET_ITEM(args, 17));
         goto exit;
     }
-    freevars = PyTuple_GET_ITEM(args, 15);
-    if (PyTuple_GET_SIZE(args) < 17) {
+    freevars = PyTuple_GET_ITEM(args, 17);
+    if (PyTuple_GET_SIZE(args) < 19) {
         goto skip_optional;
     }
-    if (!PyTuple_Check(PyTuple_GET_ITEM(args, 16))) {
-        _PyArg_BadArgument("code", "argument 17", "tuple", PyTuple_GET_ITEM(args, 16));
+    if (!PyTuple_Check(PyTuple_GET_ITEM(args, 18))) {
+        _PyArg_BadArgument("code", "argument 19", "tuple", PyTuple_GET_ITEM(args, 18));
         goto exit;
     }
-    cellvars = PyTuple_GET_ITEM(args, 16);
+    cellvars = PyTuple_GET_ITEM(args, 18);
 skip_optional:
-    return_value = code_new_impl(type, argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, linetable, exceptiontable, freevars, cellvars);
+    return_value = code_new_impl(type, argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, linetable, endlinetable, columntable, exceptiontable, freevars, cellvars);
 
 exit:
     return return_value;
@@ -151,7 +164,8 @@ PyDoc_STRVAR(code_replace__doc__,
 "        co_flags=-1, co_firstlineno=-1, co_code=None, co_consts=None,\n"
 "        co_names=None, co_varnames=None, co_freevars=None,\n"
 "        co_cellvars=None, co_filename=None, co_name=None,\n"
-"        co_linetable=None, co_exceptiontable=None)\n"
+"        co_linetable=None, co_endlinetable=None, co_columntable=None,\n"
+"        co_exceptiontable=None)\n"
 "--\n"
 "\n"
 "Return a copy of the code object with new values for the specified fields.");
@@ -168,15 +182,17 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
                   PyObject *co_varnames, PyObject *co_freevars,
                   PyObject *co_cellvars, PyObject *co_filename,
                   PyObject *co_name, PyBytesObject *co_linetable,
+                  PyBytesObject *co_endlinetable,
+                  PyBytesObject *co_columntable,
                   PyBytesObject *co_exceptiontable);
 
 static PyObject *
 code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"co_argcount", "co_posonlyargcount", "co_kwonlyargcount", "co_nlocals", "co_stacksize", "co_flags", "co_firstlineno", "co_code", "co_consts", "co_names", "co_varnames", "co_freevars", "co_cellvars", "co_filename", "co_name", "co_linetable", "co_exceptiontable", NULL};
+    static const char * const _keywords[] = {"co_argcount", "co_posonlyargcount", "co_kwonlyargcount", "co_nlocals", "co_stacksize", "co_flags", "co_firstlineno", "co_code", "co_consts", "co_names", "co_varnames", "co_freevars", "co_cellvars", "co_filename", "co_name", "co_linetable", "co_endlinetable", "co_columntable", "co_exceptiontable", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "replace", 0};
-    PyObject *argsbuf[17];
+    PyObject *argsbuf[19];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     int co_argcount = self->co_argcount;
     int co_posonlyargcount = self->co_posonlyargcount;
@@ -194,6 +210,8 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
     PyObject *co_filename = self->co_filename;
     PyObject *co_name = self->co_name;
     PyBytesObject *co_linetable = (PyBytesObject *)self->co_linetable;
+    PyBytesObject *co_endlinetable = (PyBytesObject *)self->co_endlinetable;
+    PyBytesObject *co_columntable = (PyBytesObject *)self->co_columntable;
     PyBytesObject *co_exceptiontable = (PyBytesObject *)self->co_exceptiontable;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf);
@@ -362,13 +380,33 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
             goto skip_optional_kwonly;
         }
     }
-    if (!PyBytes_Check(args[16])) {
-        _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[16]);
+    if (args[16]) {
+        if (!PyBytes_Check(args[16])) {
+            _PyArg_BadArgument("replace", "argument 'co_endlinetable'", "bytes", args[16]);
+            goto exit;
+        }
+        co_endlinetable = (PyBytesObject *)args[16];
+        if (!--noptargs) {
+            goto skip_optional_kwonly;
+        }
+    }
+    if (args[17]) {
+        if (!PyBytes_Check(args[17])) {
+            _PyArg_BadArgument("replace", "argument 'co_columntable'", "bytes", args[17]);
+            goto exit;
+        }
+        co_columntable = (PyBytesObject *)args[17];
+        if (!--noptargs) {
+            goto skip_optional_kwonly;
+        }
+    }
+    if (!PyBytes_Check(args[18])) {
+        _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[18]);
         goto exit;
     }
-    co_exceptiontable = (PyBytesObject *)args[16];
+    co_exceptiontable = (PyBytesObject *)args[18];
 skip_optional_kwonly:
-    return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_linetable, co_exceptiontable);
+    return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_linetable, co_endlinetable, co_columntable, co_exceptiontable);
 
 exit:
     return return_value;
@@ -410,4 +448,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=ba4c5487e0364ce8 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=a75c9ca013d9bf7d input=a9049054013a1b77]*/