bpo-16806: Fix `lineno` and `col_offset` for multi-line string tokens (GH-10021)

4 files changed, 37 insertions, 24 deletions

diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index db9a6ca..2c8d8ab 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -683,6 +683,25 @@ class ASTHelpers_Test(unittest.TestCase):
         node = ast.parse('async def foo():\n  x = "not docstring"')
         self.assertIsNone(ast.get_docstring(node.body[0]))
 
+    def test_multi_line_docstring_col_offset_and_lineno_issue16806(self):
+        node = ast.parse(
+            '"""line one\nline two"""\n\n'
+            'def foo():\n  """line one\n  line two"""\n\n'
+            '  def bar():\n    """line one\n    line two"""\n'
+            '  """line one\n  line two"""\n'
+            '"""line one\nline two"""\n\n'
+        )
+        self.assertEqual(node.body[0].col_offset, 0)
+        self.assertEqual(node.body[0].lineno, 1)
+        self.assertEqual(node.body[1].body[0].col_offset, 2)
+        self.assertEqual(node.body[1].body[0].lineno, 5)
+        self.assertEqual(node.body[1].body[1].body[0].col_offset, 4)
+        self.assertEqual(node.body[1].body[1].body[0].lineno, 9)
+        self.assertEqual(node.body[1].body[2].col_offset, 2)
+        self.assertEqual(node.body[1].body[2].lineno, 11)
+        self.assertEqual(node.body[2].col_offset, 0)
+        self.assertEqual(node.body[2].lineno, 13)
+
     def test_literal_eval(self):
         self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3])
         self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42})
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 9e45770..9d60be3 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -270,10 +270,7 @@ f'{a * x()} {a * x()} {a * x()}'
         self.assertEqual(binop.right.col_offset, 7)  # FIXME: this is wrong
 
     def test_ast_line_numbers_multiline_fstring(self):
-        # FIXME: This test demonstrates invalid behavior due to JoinedStr's
-        # immediate child nodes containing the wrong lineno.  The enclosed
-        # expressions have valid line information and column offsets.
-        # See bpo-16806 and bpo-30465 for details.
+        # See bpo-30465 for details.
         expr = """
 a = 10
 f'''
@@ -298,19 +295,16 @@ non-important content
         self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue)
         self.assertEqual(type(t.body[1].value.values[2]), ast.Constant)
         self.assertEqual(type(t.body[1].value.values[2].value), str)
-        # NOTE: the following invalid behavior is described in bpo-16806.
-        # - line number should be the *first* line (3), not the *last* (8)
-        # - column offset should not be -1
-        self.assertEqual(t.body[1].lineno, 8)
-        self.assertEqual(t.body[1].value.lineno, 8)
-        self.assertEqual(t.body[1].value.values[0].lineno, 8)
-        self.assertEqual(t.body[1].value.values[1].lineno, 8)
-        self.assertEqual(t.body[1].value.values[2].lineno, 8)
-        self.assertEqual(t.body[1].col_offset, -1)
-        self.assertEqual(t.body[1].value.col_offset, -1)
-        self.assertEqual(t.body[1].value.values[0].col_offset, -1)
-        self.assertEqual(t.body[1].value.values[1].col_offset, -1)
-        self.assertEqual(t.body[1].value.values[2].col_offset, -1)
+        self.assertEqual(t.body[1].lineno, 3)
+        self.assertEqual(t.body[1].value.lineno, 3)
+        self.assertEqual(t.body[1].value.values[0].lineno, 3)
+        self.assertEqual(t.body[1].value.values[1].lineno, 3)
+        self.assertEqual(t.body[1].value.values[2].lineno, 3)
+        self.assertEqual(t.body[1].col_offset, 0)
+        self.assertEqual(t.body[1].value.col_offset, 0)
+        self.assertEqual(t.body[1].value.values[0].col_offset, 0)
+        self.assertEqual(t.body[1].value.values[1].col_offset, 0)
+        self.assertEqual(t.body[1].value.values[2].col_offset, 0)
         # NOTE: the following lineno information and col_offset is correct for
         # expressions within FormattedValues.
         binop = t.body[1].value.values[1].value
@@ -321,8 +315,8 @@ non-important content
         self.assertEqual(binop.lineno, 4)
         self.assertEqual(binop.left.lineno, 4)
         self.assertEqual(binop.right.lineno, 6)
-        self.assertEqual(binop.col_offset, 3)
-        self.assertEqual(binop.left.col_offset, 3)
+        self.assertEqual(binop.col_offset, 4)
+        self.assertEqual(binop.left.col_offset, 4)
         self.assertEqual(binop.right.col_offset, 7)
 
     def test_docstring(self):
diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py
index b2a22861..527aca6 100644
--- a/Lib/test/test_opcodes.py
+++ b/Lib/test/test_opcodes.py
@@ -27,7 +27,7 @@ class OpcodeTest(unittest.TestCase):
             with open(ann_module.__file__) as f:
                 txt = f.read()
             co = compile(txt, ann_module.__file__, 'exec')
-            self.assertEqual(co.co_firstlineno, 6)
+            self.assertEqual(co.co_firstlineno, 3)
         except OSError:
             pass
 
diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py
index 55bcde4..635ba57 100644
--- a/Lib/test/test_string_literals.py
+++ b/Lib/test/test_string_literals.py
@@ -117,7 +117,7 @@ class TestLiterals(unittest.TestCase):
             eval("'''\n\\z'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].filename, '<string>')
-        self.assertEqual(w[0].lineno, 2)
+        self.assertEqual(w[0].lineno, 1)
 
         with warnings.catch_warnings(record=True) as w:
             warnings.simplefilter('error', category=SyntaxWarning)
@@ -126,7 +126,7 @@ class TestLiterals(unittest.TestCase):
             exc = cm.exception
         self.assertEqual(w, [])
         self.assertEqual(exc.filename, '<string>')
-        self.assertEqual(exc.lineno, 2)
+        self.assertEqual(exc.lineno, 1)
 
     def test_eval_str_raw(self):
         self.assertEqual(eval(""" r'x' """), 'x')
@@ -166,7 +166,7 @@ class TestLiterals(unittest.TestCase):
             eval("b'''\n\\z'''")
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].filename, '<string>')
-        self.assertEqual(w[0].lineno, 2)
+        self.assertEqual(w[0].lineno, 1)
 
         with warnings.catch_warnings(record=True) as w:
             warnings.simplefilter('error', category=SyntaxWarning)
@@ -175,7 +175,7 @@ class TestLiterals(unittest.TestCase):
             exc = cm.exception
         self.assertEqual(w, [])
         self.assertEqual(exc.filename, '<string>')
-        self.assertEqual(exc.lineno, 2)
+        self.assertEqual(exc.lineno, 1)
 
     def test_eval_bytes_raw(self):
         self.assertEqual(eval(""" br'x' """), b'x')