SF bug 430991: wrong co_lnotab

Armin Rigo pointed out that the way the line-# table got built didn't work
for lines generating more than 255 bytes of bytecode.  Fixed as he
suggested, plus corresponding changes to pyassem.py, plus added some
long overdue docs about this subtle table to compile.c.

Bugfix candidate (line numbers may be off in tracebacks under -O).

4 files changed, 83 insertions, 44 deletions

diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py
index e8810ae..0684623 100644
--- a/Lib/compiler/pyassem.py
+++ b/Lib/compiler/pyassem.py
@@ -613,16 +613,16 @@ def twobyte(val):
 class LineAddrTable:
     """lnotab
     
-    This class builds the lnotab, which is undocumented but described
-    by com_set_lineno in compile.c.  Here's an attempt at explanation:
+    This class builds the lnotab, which is documented in compile.c.
+    Here's a brief recap:
 
     For each SET_LINENO instruction after the first one, two bytes are
     added to lnotab.  (In some cases, multiple two-byte entries are
     added.)  The first byte is the distance in bytes between the
     instruction for the last SET_LINENO and the current SET_LINENO.
     The second byte is offset in line numbers.  If either offset is
-    greater than 255, multiple two-byte entries are added -- one entry
-    for each factor of 255.
+    greater than 255, multiple two-byte entries are added -- see
+    compile.c for the delicate details.
     """
 
     def __init__(self):
@@ -657,19 +657,16 @@ class LineAddrTable:
             # compiler because it only generates a SET_LINENO instruction
             # for the assignment.
             if line > 0:
-                while addr > 0 or line > 0:
-                    # write the values in 1-byte chunks that sum
-                    # to desired value
-                    trunc_addr = addr
-                    trunc_line = line
-                    if trunc_addr > 255:
-                        trunc_addr = 255
-                    if trunc_line > 255:
-                        trunc_line = 255
-                    self.lnotab.append(trunc_addr)
-                    self.lnotab.append(trunc_line)
-                    addr = addr - trunc_addr
-                    line = line - trunc_line
+                push = self.lnotab.append
+                while addr > 255:
+                    push(255); push(0)
+                    addr -= 255
+                while line > 255:
+                    push(addr); push(255)
+                    line -= 255
+                    addr = 0
+                if addr > 0 or line > 0:
+                    push(addr); push(line)
                 self.lastline = lineno
                 self.lastoff = self.codeOffset
 
diff --git a/Misc/ACKS b/Misc/ACKS
index 3326096..ebe6770 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -319,6 +319,7 @@ Michael P. Reilly
 Bernhard Reiter
 Steven Reiz
 Jan Pieter Riegel
+Armin Rigo
 Nicholas Riley
 Jean-Claude Rimbault
 Andy Robinson
diff --git a/Python/compile.c b/Python/compile.c
index ec3de26..b9ba0fc 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -336,6 +336,50 @@ PyCode_New(int argcount, int nlocals, int stacksize, int flags,
    c_argcount, c_globals, and c_flags.
 */
 
+/* All about c_lnotab.
+
+c_lnotab is an array of unsigned bytes disguised as a Python string.  In -O
+mode, SET_LINENO opcodes aren't generated, and bytecode offsets are mapped
+to source code line #s (when needed for tracebacks) via c_lnotab instead.
+The array is conceptually a list of
+    (bytecode offset increment, line number increment)
+pairs.  The details are important and delicate, best illustrated by example:
+
+    byte code offset    source code line number
+        0		    1
+        6		    2
+       50		    7
+      350                 307
+      361                 308
+
+The first trick is that these numbers aren't stored, only the increments
+from one row to the next (this doesn't really work, but it's a start):
+
+    0, 1,  6, 1,  44, 5,  300, 300,  11, 1
+
+The second trick is that an unsigned byte can't hold negative values, or
+values larger than 255, so (a) there's a deep assumption that byte code
+offsets and their corresponding line #s both increase monotonically, and (b)
+if at least one column jumps by more than 255 from one row to the next, more
+than one pair is written to the table. In case #b, there's no way to know
+from looking at the table later how many were written.  That's the delicate
+part.  A user of c_lnotab desiring to find the source line number
+corresponding to a bytecode address A should do something like this
+
+    lineno = addr = 0
+    for addr_incr, line_incr in c_lnotab:
+        addr += addr_incr
+        if addr > A:
+            return lineno
+        lineno += line_incr
+
+In order for this to work, when the addr field increments by more than 255,
+the line # increment in each pair generated must be 0 until the remaining addr
+increment is < 256.  So, in the example above, com_set_lineno should not (as
+was actually done until 2.2) expand 300, 300 to 255, 255,  45, 45, but to
+255, 0,  45, 255,  0, 45.
+*/
+
 struct compiling {
 	PyObject *c_code;	/* string */
 	PyObject *c_consts;	/* list of objects */
@@ -692,17 +736,17 @@ com_set_lineno(struct compiling *c, int lineno)
 	else {
 		int incr_addr = c->c_nexti - c->c_last_addr;
 		int incr_line = lineno - c->c_last_line;
-		while (incr_addr > 0 || incr_line > 0) {
-			int trunc_addr = incr_addr;
-			int trunc_line = incr_line;
-			if (trunc_addr > 255)
-				trunc_addr = 255;
-			if (trunc_line > 255)
-				trunc_line = 255;
-			com_add_lnotab(c, trunc_addr, trunc_line);
-			incr_addr -= trunc_addr;
-			incr_line -= trunc_line;
+		while (incr_addr > 255) {
+			com_add_lnotab(c, 255, 0);
+			incr_addr -= 255;
+		}
+		while (incr_line > 255) {
+			com_add_lnotab(c, incr_addr, 255);
+			incr_line -=255;
+			incr_addr = 0;
 		}
+		if (incr_addr > 0 || incr_line > 0)
+			com_add_lnotab(c, incr_addr, incr_line);
 		c->c_last_addr = c->c_nexti;
 		c->c_last_line = lineno;
 	}
diff --git a/Tools/compiler/compiler/pyassem.py b/Tools/compiler/compiler/pyassem.py
index e8810ae..0684623 100644
--- a/Tools/compiler/compiler/pyassem.py
+++ b/Tools/compiler/compiler/pyassem.py
@@ -613,16 +613,16 @@ def twobyte(val):
 class LineAddrTable:
     """lnotab
     
-    This class builds the lnotab, which is undocumented but described
-    by com_set_lineno in compile.c.  Here's an attempt at explanation:
+    This class builds the lnotab, which is documented in compile.c.
+    Here's a brief recap:
 
     For each SET_LINENO instruction after the first one, two bytes are
     added to lnotab.  (In some cases, multiple two-byte entries are
     added.)  The first byte is the distance in bytes between the
     instruction for the last SET_LINENO and the current SET_LINENO.
     The second byte is offset in line numbers.  If either offset is
-    greater than 255, multiple two-byte entries are added -- one entry
-    for each factor of 255.
+    greater than 255, multiple two-byte entries are added -- see
+    compile.c for the delicate details.
     """
 
     def __init__(self):
@@ -657,19 +657,16 @@ class LineAddrTable:
             # compiler because it only generates a SET_LINENO instruction
             # for the assignment.
             if line > 0:
-                while addr > 0 or line > 0:
-                    # write the values in 1-byte chunks that sum
-                    # to desired value
-                    trunc_addr = addr
-                    trunc_line = line
-                    if trunc_addr > 255:
-                        trunc_addr = 255
-                    if trunc_line > 255:
-                        trunc_line = 255
-                    self.lnotab.append(trunc_addr)
-                    self.lnotab.append(trunc_line)
-                    addr = addr - trunc_addr
-                    line = line - trunc_line
+                push = self.lnotab.append
+                while addr > 255:
+                    push(255); push(0)
+                    addr -= 255
+                while line > 255:
+                    push(addr); push(255)
+                    line -= 255
+                    addr = 0
+                if addr > 0 or line > 0:
+                    push(addr); push(line)
                 self.lastline = lineno
                 self.lastoff = self.codeOffset