bpo-16806: Fix `lineno` and `col_offset` for multi-line string tokens (GH-10021)

4 files changed, 28 insertions, 24 deletions

diff --git a/Python/ast.c b/Python/ast.c
index 69dfe3c..d71f44a 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -4284,9 +4284,13 @@ fstring_fix_node_location(const node *parent, node *n, char *expr_str)
                 start--;
             }
             cols += (int)(substr - start);
-            /* Fix lineno in mulitline strings. */
-            while ((substr = strchr(substr + 1, '\n')))
-                lines--;
+            /* adjust the start based on the number of newlines encountered
+               before the f-string expression */
+            for (char* p = parent->n_str; p < substr; p++) {
+                if (*p == '\n') {
+                    lines++;
+                }
+            }
         }
     }
     fstring_shift_node_locations(n, lines, cols);
diff --git a/Python/importlib.h b/Python/importlib.h
index 5c38196..dd78c0b 100644
--- a/Python/importlib.h
+++ b/Python/importlib.h
@@ -204,7 +204,7 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
     0,0,114,38,0,0,0,114,39,0,0,0,114,48,0,0,
     0,114,10,0,0,0,114,10,0,0,0,114,10,0,0,0,
     114,11,0,0,0,114,20,0,0,0,52,0,0,0,115,12,
-    0,0,0,8,4,4,2,8,8,8,12,8,25,8,13,114,
+    0,0,0,8,1,4,5,8,8,8,12,8,25,8,13,114,
     20,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,
     0,2,0,0,0,64,0,0,0,115,48,0,0,0,101,0,
     90,1,100,0,90,2,100,1,90,3,100,2,100,3,132,0,
@@ -255,8 +255,8 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
     0,0,0,114,3,0,0,0,114,31,0,0,0,114,38,0,
     0,0,114,39,0,0,0,114,48,0,0,0,114,10,0,0,
     0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,
-    114,49,0,0,0,120,0,0,0,115,10,0,0,0,8,2,
-    4,2,8,4,8,4,8,5,114,49,0,0,0,99,0,0,
+    114,49,0,0,0,120,0,0,0,115,10,0,0,0,8,1,
+    4,3,8,4,8,4,8,5,114,49,0,0,0,99,0,0,
     0,0,0,0,0,0,0,0,0,0,2,0,0,0,64,0,
     0,0,115,36,0,0,0,101,0,90,1,100,0,90,2,100,
     1,100,2,132,0,90,3,100,3,100,4,132,0,90,4,100,
@@ -730,7 +730,7 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
     114,123,0,0,0,218,6,115,101,116,116,101,114,114,130,0,
     0,0,114,124,0,0,0,114,10,0,0,0,114,10,0,0,
     0,114,10,0,0,0,114,11,0,0,0,114,112,0,0,0,
-    49,1,0,0,115,32,0,0,0,8,35,4,2,4,1,2,
+    49,1,0,0,115,32,0,0,0,8,1,4,36,4,1,2,
     255,12,12,8,10,8,12,2,1,10,8,4,1,10,3,2,
     1,10,7,2,1,10,3,4,1,114,112,0,0,0,169,2,
     114,113,0,0,0,114,115,0,0,0,99,2,0,0,0,2,
@@ -1147,7 +1147,7 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
     0,0,0,114,169,0,0,0,114,115,0,0,0,114,97,0,
     0,0,114,153,0,0,0,114,10,0,0,0,114,10,0,0,
     0,114,10,0,0,0,114,11,0,0,0,114,158,0,0,0,
-    195,2,0,0,115,42,0,0,0,8,7,4,2,2,1,10,
+    195,2,0,0,115,42,0,0,0,8,2,4,7,2,1,10,
     8,2,1,12,8,2,1,12,11,2,1,10,7,2,1,10,
     4,2,1,2,1,12,4,2,1,2,1,12,4,2,1,2,
     1,12,4,114,158,0,0,0,99,0,0,0,0,0,0,0,
@@ -1280,7 +1280,7 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
     0,114,90,0,0,0,114,168,0,0,0,114,169,0,0,0,
     114,115,0,0,0,114,10,0,0,0,114,10,0,0,0,114,
     10,0,0,0,114,11,0,0,0,114,172,0,0,0,12,3,
-    0,0,115,44,0,0,0,8,7,4,2,2,1,10,8,2,
+    0,0,115,44,0,0,0,8,2,4,7,2,1,10,8,2,
     1,12,6,2,1,12,8,2,1,10,3,2,1,10,8,2,
     1,10,8,2,1,2,1,12,4,2,1,2,1,12,4,2,
     1,2,1,114,172,0,0,0,99,0,0,0,0,0,0,0,
@@ -1757,8 +1757,8 @@ const unsigned char _Py_M__importlib_bootstrap[] = {
     0,114,215,0,0,0,114,218,0,0,0,114,219,0,0,0,
     114,223,0,0,0,114,224,0,0,0,114,226,0,0,0,114,
     10,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,
-    0,0,0,218,8,60,109,111,100,117,108,101,62,8,0,0,
-    0,115,94,0,0,0,4,17,4,2,8,8,8,8,4,2,
+    0,0,0,218,8,60,109,111,100,117,108,101,62,1,0,0,
+    0,115,94,0,0,0,4,24,4,2,8,8,8,8,4,2,
     4,3,16,4,14,68,14,21,14,16,8,37,8,17,8,11,
     14,8,8,11,8,12,8,16,8,36,14,101,16,26,10,45,
     14,72,8,17,8,17,8,30,8,37,8,42,8,15,14,73,
diff --git a/Python/importlib_external.h b/Python/importlib_external.h
index 791bfc4..ca83eb5 100644
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h
@@ -1137,8 +1137,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,0,114,127,0,0,0,114,182,0,0,0,114,212,0,0,
     0,114,217,0,0,0,114,220,0,0,0,114,3,0,0,0,
     114,3,0,0,0,114,3,0,0,0,114,6,0,0,0,114,
-    208,0,0,0,243,2,0,0,115,10,0,0,0,8,3,4,
-    2,8,8,8,3,8,8,114,208,0,0,0,99,0,0,0,
+    208,0,0,0,243,2,0,0,115,10,0,0,0,8,2,4,
+    3,8,8,8,3,8,8,114,208,0,0,0,99,0,0,0,
     0,0,0,0,0,0,0,0,0,3,0,0,0,64,0,0,
     0,115,74,0,0,0,101,0,90,1,100,0,90,2,100,1,
     100,2,132,0,90,3,100,3,100,4,132,0,90,4,100,5,
@@ -1236,7 +1236,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     32,32,32,78,114,3,0,0,0,41,3,114,119,0,0,0,
     114,44,0,0,0,114,26,0,0,0,114,3,0,0,0,114,
     3,0,0,0,114,6,0,0,0,114,225,0,0,0,50,3,
-    0,0,115,2,0,0,0,0,4,122,21,83,111,117,114,99,
+    0,0,115,2,0,0,0,0,1,122,21,83,111,117,114,99,
     101,76,111,97,100,101,114,46,115,101,116,95,100,97,116,97,
     99,2,0,0,0,0,0,0,0,5,0,0,0,10,0,0,
     0,67,0,0,0,115,82,0,0,0,124,0,160,0,124,1,
@@ -1520,7 +1520,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     1,0,0,90,13,95,95,99,108,97,115,115,99,101,108,108,
     95,95,114,3,0,0,0,114,3,0,0,0,114,249,0,0,
     0,114,6,0,0,0,114,239,0,0,0,160,3,0,0,115,
-    30,0,0,0,8,3,4,2,8,6,8,4,8,3,2,1,
+    30,0,0,0,8,2,4,3,8,6,8,4,8,3,2,1,
     14,11,2,1,10,4,8,7,2,1,10,5,8,4,8,6,
     8,6,114,239,0,0,0,99,0,0,0,0,0,0,0,0,
     0,0,0,0,3,0,0,0,64,0,0,0,115,46,0,0,
@@ -1768,7 +1768,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     114,213,0,0,0,114,229,0,0,0,114,136,0,0,0,114,
     179,0,0,0,114,3,0,0,0,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,114,15,1,0,0,46,4,0,
-    0,115,22,0,0,0,8,6,4,2,8,4,8,4,8,3,
+    0,115,22,0,0,0,8,2,4,6,8,4,8,4,8,3,
     8,8,8,6,8,6,8,4,8,4,2,1,114,15,1,0,
     0,99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,
     0,0,64,0,0,0,115,96,0,0,0,101,0,90,1,100,
@@ -1913,7 +1913,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     114,36,1,0,0,114,37,1,0,0,114,40,1,0,0,114,
     186,0,0,0,114,3,0,0,0,114,3,0,0,0,114,3,
     0,0,0,114,6,0,0,0,114,22,1,0,0,99,4,0,
-    0,115,22,0,0,0,8,5,4,2,8,6,8,10,8,4,
+    0,115,22,0,0,0,8,1,4,6,8,6,8,10,8,4,
     8,13,8,3,8,3,8,3,8,3,8,3,114,22,1,0,
     0,99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,
     0,0,64,0,0,0,115,80,0,0,0,101,0,90,1,100,
@@ -2462,7 +2462,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,114,65,1,0,0,114,207,0,0,0,114,72,1,0,0,
     114,37,1,0,0,114,3,0,0,0,114,3,0,0,0,114,
     3,0,0,0,114,6,0,0,0,114,57,1,0,0,79,5,
-    0,0,115,22,0,0,0,8,7,4,2,8,14,8,4,4,
+    0,0,115,22,0,0,0,8,2,4,7,8,14,8,4,4,
     2,8,12,8,5,10,48,8,31,2,1,10,17,114,57,1,
     0,0,99,4,0,0,0,0,0,0,0,6,0,0,0,8,
     0,0,0,67,0,0,0,115,146,0,0,0,124,0,160,0,
@@ -2641,8 +2641,8 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = {
     0,114,57,1,0,0,114,77,1,0,0,114,184,0,0,0,
     114,85,1,0,0,114,87,1,0,0,114,3,0,0,0,114,
     3,0,0,0,114,3,0,0,0,114,6,0,0,0,218,8,
-    60,109,111,100,117,108,101,62,8,0,0,0,115,126,0,0,
-    0,4,15,4,1,4,1,2,1,2,255,4,4,8,17,8,
+    60,109,111,100,117,108,101,62,1,0,0,0,115,126,0,0,
+    0,4,22,4,1,4,1,2,1,2,255,4,4,8,17,8,
     5,8,5,8,6,8,6,8,12,8,10,8,9,8,5,8,
     7,8,9,12,22,10,127,0,7,16,1,12,2,4,1,4,
     2,6,2,6,2,8,2,18,71,8,40,8,19,8,12,8,
diff --git a/Python/importlib_zipimport.h b/Python/importlib_zipimport.h
index e00010c..299d1c5 100644
--- a/Python/importlib_zipimport.h
+++ b/Python/importlib_zipimport.h
@@ -484,7 +484,7 @@ const unsigned char _Py_M__zipimport[] = {
     64,0,0,0,114,65,0,0,0,114,78,0,0,0,114,82,
     0,0,0,114,83,0,0,0,114,9,0,0,0,114,9,0,
     0,0,114,9,0,0,0,114,10,0,0,0,114,4,0,0,
-    0,45,0,0,0,115,24,0,0,0,8,13,4,5,8,46,
+    0,45,0,0,0,115,24,0,0,0,8,1,4,17,8,46,
     10,32,10,12,8,10,8,21,8,11,8,26,8,13,8,38,
     8,18,122,12,95,95,105,110,105,116,95,95,46,112,121,99,
     84,114,60,0,0,0,70,41,3,122,4,46,112,121,99,84,
@@ -1044,7 +1044,7 @@ const unsigned char _Py_M__zipimport[] = {
     34,0,0,0,114,182,0,0,0,114,183,0,0,0,114,184,
     0,0,0,114,189,0,0,0,114,9,0,0,0,114,9,0,
     0,0,114,9,0,0,0,114,10,0,0,0,114,80,0,0,
-    0,212,2,0,0,115,14,0,0,0,8,5,4,1,4,2,
+    0,212,2,0,0,115,14,0,0,0,8,1,4,5,4,2,
     8,4,8,9,8,6,8,11,114,80,0,0,0,41,45,114,
     84,0,0,0,90,26,95,102,114,111,122,101,110,95,105,109,
     112,111,114,116,108,105,98,95,101,120,116,101,114,110,97,108,
@@ -1065,8 +1065,8 @@ const unsigned char _Py_M__zipimport[] = {
     0,0,114,170,0,0,0,114,152,0,0,0,114,150,0,0,
     0,114,44,0,0,0,114,80,0,0,0,114,9,0,0,0,
     114,9,0,0,0,114,9,0,0,0,114,10,0,0,0,218,
-    8,60,109,111,100,117,108,101,62,13,0,0,0,115,88,0,
-    0,0,4,4,8,1,16,1,8,1,8,1,8,1,8,1,
+    8,60,109,111,100,117,108,101,62,1,0,0,0,115,88,0,
+    0,0,4,16,8,1,16,1,8,1,8,1,8,1,8,1,
     8,1,8,2,8,3,6,1,14,3,16,4,4,2,8,2,
     4,1,4,1,4,2,14,127,0,127,0,1,12,1,12,1,
     2,1,2,252,4,9,8,4,8,9,8,31,8,126,2,254,