diff options
| author | Marta Gómez Macías <mgmacias@google.com> | 2023-05-28 14:15:53 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-05-28 14:15:53 (GMT) |
| commit | 96fff35325e519cc76ffacf22e57e4c393d4446f (patch) | |
| tree | 56d287fb561e70c42a79c9be294d744d478efe57 /Python/Python-tokenize.c | |
| parent | 3821b92c1faf7e7058feeb0048511c946a841105 (diff) | |
| download | cpython-96fff35325e519cc76ffacf22e57e4c393d4446f.zip cpython-96fff35325e519cc76ffacf22e57e4c393d4446f.tar.gz cpython-96fff35325e519cc76ffacf22e57e4c393d4446f.tar.bz2 | |
gh-105017: Include CRLF lines in strings and column numbers (#105030)
Co-authored-by: Pablo Galindo <pablogsal@gmail.com>
Diffstat (limited to 'Python/Python-tokenize.c')
| -rw-r--r-- | Python/Python-tokenize.c | 9 |
1 files changed, 7 insertions, 2 deletions
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 01c2215..4eced66 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -55,7 +55,7 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source, if (filename == NULL) { return NULL; } - self->tok = _PyTokenizer_FromUTF8(source, 1); + self->tok = _PyTokenizer_FromUTF8(source, 1, 1); if (self->tok == NULL) { Py_DECREF(filename); return NULL; @@ -240,7 +240,12 @@ tokenizeriter_next(tokenizeriterobject *it) type = NAME; } else if (type == NEWLINE) { - str = PyUnicode_FromString("\n"); + Py_DECREF(str); + if (it->tok->start[0] == '\r') { + str = PyUnicode_FromString("\r\n"); + } else { + str = PyUnicode_FromString("\n"); + } end_col_offset++; } } |