bpo-42739: Don't use sentinels to mark end of line table. (GH-25657)

* Add length parameter to PyLineTable_InitAddressRange and doen't use sentinel values at end of table. Makes the line number table more robust.

* Update PyCodeAddressRange to match PEP 626.

2 files changed, 21 insertions, 24 deletions

diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index 9bb49f1..c76ac90 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -456,15 +456,15 @@ code_getlnotab(PyCodeObject *code, void *closure)
     }
     _PyCode_InitAddressRange(code, &bounds);
     while (PyLineTable_NextAddressRange(&bounds)) {
-        if (bounds.ar_computed_line != line) {
+        if (bounds.opaque.computed_line != line) {
             int bdelta = bounds.ar_start - code_offset;
-            int ldelta = bounds.ar_computed_line - line;
+            int ldelta = bounds.opaque.computed_line - line;
             if (!emit_delta(&bytes, bdelta, ldelta, &table_offset)) {
                 Py_DECREF(bytes);
                 return NULL;
             }
             code_offset = bounds.ar_start;
-            line = bounds.ar_computed_line;
+            line = bounds.opaque.computed_line;
         }
     }
     _PyBytes_Resize(&bytes, table_offset);
@@ -1120,20 +1120,20 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args))
 static void
 retreat(PyCodeAddressRange *bounds)
 {
-    int ldelta = ((signed char *)bounds->lo_next)[-1];
+    int ldelta = ((signed char *)bounds->opaque.lo_next)[-1];
     if (ldelta == -128) {
         ldelta = 0;
     }
-    bounds->ar_computed_line -= ldelta;
-    bounds->lo_next -= 2;
+    bounds->opaque.computed_line -= ldelta;
+    bounds->opaque.lo_next -= 2;
     bounds->ar_end = bounds->ar_start;
-    bounds->ar_start -= ((unsigned char *)bounds->lo_next)[-2];
-    ldelta = ((signed char *)bounds->lo_next)[-1];
+    bounds->ar_start -= ((unsigned char *)bounds->opaque.lo_next)[-2];
+    ldelta = ((signed char *)bounds->opaque.lo_next)[-1];
     if (ldelta == -128) {
         bounds->ar_line = -1;
     }
     else {
-        bounds->ar_line = bounds->ar_computed_line;
+        bounds->ar_line = bounds->opaque.computed_line;
     }
 }
 
@@ -1141,23 +1141,22 @@ static void
 advance(PyCodeAddressRange *bounds)
 {
     bounds->ar_start = bounds->ar_end;
-    int delta = ((unsigned char *)bounds->lo_next)[0];
-    assert (delta < 255);
+    int delta = ((unsigned char *)bounds->opaque.lo_next)[0];
     bounds->ar_end += delta;
-    int ldelta = ((signed char *)bounds->lo_next)[1];
-    bounds->lo_next += 2;
+    int ldelta = ((signed char *)bounds->opaque.lo_next)[1];
+    bounds->opaque.lo_next += 2;
     if (ldelta == -128) {
         bounds->ar_line = -1;
     }
     else {
-        bounds->ar_computed_line += ldelta;
-        bounds->ar_line = bounds->ar_computed_line;
+        bounds->opaque.computed_line += ldelta;
+        bounds->ar_line = bounds->opaque.computed_line;
     }
 }
 
 static inline int
 at_end(PyCodeAddressRange *bounds) {
-    return ((unsigned char *)bounds->lo_next)[0] == 255;
+    return bounds->opaque.lo_next >= bounds->opaque.limit;
 }
 
 int
@@ -1256,12 +1255,13 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq)
 }
 
 void
-PyLineTable_InitAddressRange(char *linetable, int firstlineno, PyCodeAddressRange *range)
+PyLineTable_InitAddressRange(char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range)
 {
-    range->lo_next = linetable;
+    range->opaque.lo_next = linetable;
+    range->opaque.limit = range->opaque.lo_next + length;
     range->ar_start = -1;
     range->ar_end = 0;
-    range->ar_computed_line = firstlineno;
+    range->opaque.computed_line = firstlineno;
     range->ar_line = -1;
 }
 
@@ -1269,7 +1269,8 @@ int
 _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds)
 {
     char *linetable = PyBytes_AS_STRING(co->co_linetable);
-    PyLineTable_InitAddressRange(linetable, co->co_firstlineno, bounds);
+    Py_ssize_t length = PyBytes_GET_SIZE(co->co_linetable);
+    PyLineTable_InitAddressRange(linetable, length, co->co_firstlineno, bounds);
     return bounds->ar_line;
 }
 
diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt
index 59c6525..f482310 100644
--- a/Objects/lnotab_notes.txt
+++ b/Objects/lnotab_notes.txt
@@ -39,7 +39,6 @@ Note that the end - start value is always positive.
 Finally, in order to fit into a single byte we need to convert start deltas to the range 0 <= delta <= 254,
 and line deltas to the range -127  <= delta <= 127.
 A line delta of -128 is used to indicate no line number.
-A start delta of 255 is used as a sentinel to mark the end of the table.
 Also note that a delta of zero indicates that there are no bytecodes in the given range,
 which means we can use an invalid line number for that range.
 
@@ -54,7 +53,6 @@ Final form:
     16              +1
     0               +127 (line 135, but the range is empty as no bytecodes are at line 135)
     4               +73
-    255 (end mark)  ---
 
 Iterating over the table.
 -------------------------
@@ -68,8 +66,6 @@ def co_lines(code):
     end = 0
     table_iter = iter(code.internal_line_table):
     for sdelta, ldelta in table_iter:
-        if sdelta == 255:
-            break
         if ldelta == 0: # No change to line number, just accumulate changes to end
             end += odelta
             continue