diff options
| author | Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> | 2023-05-28 14:16:43 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-05-28 14:16:43 (GMT) |
| commit | 3f8d5d9ed60ae0f70c92dc29587cc6367eda80c3 (patch) | |
| tree | 4ec62b8e48ea5c86d332d022982d69a054bf584f /Python/Python-tokenize.c | |
| parent | 41b622b1e8736bd86c8dc2e1471121d5f6a88937 (diff) | |
| download | cpython-3f8d5d9ed60ae0f70c92dc29587cc6367eda80c3.zip cpython-3f8d5d9ed60ae0f70c92dc29587cc6367eda80c3.tar.gz cpython-3f8d5d9ed60ae0f70c92dc29587cc6367eda80c3.tar.bz2 | |
[3.12] gh-105017: Include CRLF lines in strings and column numbers (GH-105030) (#105041)
gh-105017: Include CRLF lines in strings and column numbers (GH-105030)
(cherry picked from commit 96fff35325e519cc76ffacf22e57e4c393d4446f)
Co-authored-by: Marta Gómez Macías <mgmacias@google.com>
Co-authored-by: Pablo Galindo <pablogsal@gmail.com>
Diffstat (limited to 'Python/Python-tokenize.c')
| -rw-r--r-- | Python/Python-tokenize.c | 9 |
1 files changed, 7 insertions, 2 deletions
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 01c2215..4eced66 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -55,7 +55,7 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source, if (filename == NULL) { return NULL; } - self->tok = _PyTokenizer_FromUTF8(source, 1); + self->tok = _PyTokenizer_FromUTF8(source, 1, 1); if (self->tok == NULL) { Py_DECREF(filename); return NULL; @@ -240,7 +240,12 @@ tokenizeriter_next(tokenizeriterobject *it) type = NAME; } else if (type == NEWLINE) { - str = PyUnicode_FromString("\n"); + Py_DECREF(str); + if (it->tok->start[0] == '\r') { + str = PyUnicode_FromString("\r\n"); + } else { + str = PyUnicode_FromString("\n"); + } end_col_offset++; } } |