diff options
| author | Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> | 2023-06-09 16:58:14 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-06-09 16:58:14 (GMT) |
| commit | 16b1cdc87c08c01294b66257a26574725b005c50 (patch) | |
| tree | 07f78346e497a5e8691556904c8cf3bddb574f63 /Python/Python-tokenize.c | |
| parent | 97d846dc2b6a790298cbfbb5669d180281cfda89 (diff) | |
| download | cpython-16b1cdc87c08c01294b66257a26574725b005c50.zip cpython-16b1cdc87c08c01294b66257a26574725b005c50.tar.gz cpython-16b1cdc87c08c01294b66257a26574725b005c50.tar.bz2 | |
[3.12] gh-105564: Don't include artificial newlines in the line attribute of tokens (GH-105565) (#105579)
Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
Diffstat (limited to 'Python/Python-tokenize.c')
| -rw-r--r-- | Python/Python-tokenize.c | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 2cf052a..1938562 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -206,6 +206,9 @@ tokenizeriter_next(tokenizeriterobject *it) line = PyUnicode_FromString(""); } else { Py_ssize_t size = it->tok->inp - line_start; + if (size >= 1 && it->tok->implicit_newline) { + size -= 1; + } line = PyUnicode_DecodeUTF8(line_start, size, "replace"); } if (line == NULL) { |