summaryrefslogtreecommitdiffstats
path: root/Python
diff options
context:
space:
mode:
authorPablo Galindo Salgado <Pablogsal@gmail.com>2023-06-09 16:01:26 (GMT)
committerGitHub <noreply@github.com>2023-06-09 16:01:26 (GMT)
commitd7f46bcd989580340675bf0a9fdbfa1505a37e81 (patch)
treecb1b8cce1ae866cf8c6c053802c3f47542539fde /Python
parent1dd267af642ed6df05a1c106e9dafb8252d826e6 (diff)
downloadcpython-d7f46bcd989580340675bf0a9fdbfa1505a37e81.zip
cpython-d7f46bcd989580340675bf0a9fdbfa1505a37e81.tar.gz
cpython-d7f46bcd989580340675bf0a9fdbfa1505a37e81.tar.bz2
gh-105564: Don't include artificial newlines in the line attribute of tokens (#105565)
Diffstat (limited to 'Python')
-rw-r--r--Python/Python-tokenize.c3
1 files changed, 3 insertions, 0 deletions
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 2cf052a..1938562 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -206,6 +206,9 @@ tokenizeriter_next(tokenizeriterobject *it)
line = PyUnicode_FromString("");
} else {
Py_ssize_t size = it->tok->inp - line_start;
+ if (size >= 1 && it->tok->implicit_newline) {
+ size -= 1;
+ }
line = PyUnicode_DecodeUTF8(line_start, size, "replace");
}
if (line == NULL) {