diff options
| author | Pablo Galindo Salgado <Pablogsal@gmail.com> | 2023-06-09 16:01:26 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-06-09 16:01:26 (GMT) |
| commit | d7f46bcd989580340675bf0a9fdbfa1505a37e81 (patch) | |
| tree | cb1b8cce1ae866cf8c6c053802c3f47542539fde /Python/Python-tokenize.c | |
| parent | 1dd267af642ed6df05a1c106e9dafb8252d826e6 (diff) | |
| download | cpython-d7f46bcd989580340675bf0a9fdbfa1505a37e81.zip cpython-d7f46bcd989580340675bf0a9fdbfa1505a37e81.tar.gz cpython-d7f46bcd989580340675bf0a9fdbfa1505a37e81.tar.bz2 | |
gh-105564: Don't include artificial newlines in the line attribute of tokens (#105565)
Diffstat (limited to 'Python/Python-tokenize.c')
| -rw-r--r-- | Python/Python-tokenize.c | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 2cf052a..1938562 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -206,6 +206,9 @@ tokenizeriter_next(tokenizeriterobject *it) line = PyUnicode_FromString(""); } else { Py_ssize_t size = it->tok->inp - line_start; + if (size >= 1 && it->tok->implicit_newline) { + size -= 1; + } line = PyUnicode_DecodeUTF8(line_start, size, "replace"); } if (line == NULL) { |