[3.12] gh-120343: Do not reset byte_col_offset_diff after multiline tokens (GH-120352) (#120356)

(cherry picked from commit 1b62bcee941e54244b3ce6476aef8913604987c9)

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>

1 files changed, 6 insertions, 1 deletions

diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 664e7d8..ebcd9ce 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -35,6 +35,7 @@ typedef struct
     /* Needed to cache line for performance */
     PyObject *last_line;
     Py_ssize_t last_lineno;
+    Py_ssize_t last_end_lineno;
     Py_ssize_t byte_col_offset_diff;
 } tokenizeriterobject;
 
@@ -76,6 +77,7 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
     self->last_line = NULL;
     self->byte_col_offset_diff = 0;
     self->last_lineno = 0;
+    self->last_end_lineno = 0;
 
     return (PyObject *)self;
 }
@@ -226,7 +228,9 @@ tokenizeriter_next(tokenizeriterobject *it)
             Py_XDECREF(it->last_line);
             line = PyUnicode_DecodeUTF8(line_start, size, "replace");
             it->last_line = line;
-            it->byte_col_offset_diff = 0;
+            if (it->tok->lineno != it->last_end_lineno) {
+                it->byte_col_offset_diff = 0;
+            }
         } else {
             // Line hasn't changed so we reuse the cached one.
             line = it->last_line;
@@ -240,6 +244,7 @@ tokenizeriter_next(tokenizeriterobject *it)
     Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
     Py_ssize_t end_lineno = it->tok->lineno;
     it->last_lineno = lineno;
+    it->last_end_lineno = end_lineno;
 
     Py_ssize_t col_offset = -1;
     Py_ssize_t end_col_offset = -1;