diff options
| -rw-r--r-- | Lib/test/test_tabnanny.py | 2 | ||||
| -rw-r--r-- | Lib/test/test_tokenize.py | 11 | ||||
| -rw-r--r-- | Python/Python-tokenize.c | 15 |
3 files changed, 20 insertions, 8 deletions
diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index dac4731..aa70011 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -317,7 +317,7 @@ class TestCommandLine(TestCase): with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: stderr = f"{file_path!r}: Token Error: " stderr += ('unindent does not match any outer indentation level' - ' (<tokenize>, line 3)') + ' (<string>, line 3)') self.validate_cmd(file_path, stderr=stderr, expect_failure=True) def test_with_error_free_file(self): diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index dda7243..8e7ab3d 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -92,9 +92,18 @@ def k(x): readline = BytesIO(indent_error_file).readline with self.assertRaisesRegex(IndentationError, "unindent does not match any " - "outer indentation level"): + "outer indentation level") as e: for tok in tokenize(readline): pass + self.assertEqual(e.exception.lineno, 3) + self.assertEqual(e.exception.filename, '<string>') + self.assertEqual(e.exception.end_lineno, None) + self.assertEqual(e.exception.end_offset, None) + self.assertEqual( + e.exception.msg, + 'unindent does not match any outer indentation level') + self.assertEqual(e.exception.offset, 9) + self.assertEqual(e.exception.text, ' x += 5\n') def test_int(self): # Ordinary integers and binary operators diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 43b44be..f7e32d3 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -89,11 +89,9 @@ _tokenizer_error(struct tok_state *tok) } return -1; case E_DEDENT: - PyErr_Format(PyExc_IndentationError, - "unindent does not match any outer indentation level " - "(<tokenize>, line %d)", - tok->lineno); - return -1; + msg = "unindent does not match any outer indentation level"; + errtype = PyExc_IndentationError; + break; case E_INTR: if (!PyErr_Occurred()) { PyErr_SetNone(PyExc_KeyboardInterrupt); @@ -131,7 +129,12 @@ _tokenizer_error(struct tok_state *tok) goto exit; } - tmp = Py_BuildValue("(OnnOii)", tok->filename, tok->lineno, 0, error_line, 0, 0); + Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, tok->inp - tok->buf); + if (offset == -1) { + result = -1; + goto exit; + } + tmp = Py_BuildValue("(OnnOOO)", tok->filename, tok->lineno, offset, error_line, Py_None, Py_None); if (!tmp) { result = -1; goto exit; |