diff options
| author | Pablo Galindo <Pablogsal@gmail.com> | 2020-04-21 00:53:04 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-04-21 00:53:04 (GMT) |
| commit | 11a7f158ef51b0edcde3c3d9215172354e385877 (patch) | |
| tree | 3bb7e125dce1a522ccbbe1ffbd14204c71bc852e /Parser/parsetok.c | |
| parent | 6a9e80a93148b13e4d3bceaab5ea1804ab0e64d5 (diff) | |
| download | cpython-11a7f158ef51b0edcde3c3d9215172354e385877.zip cpython-11a7f158ef51b0edcde3c3d9215172354e385877.tar.gz cpython-11a7f158ef51b0edcde3c3d9215172354e385877.tar.bz2 | |
bpo-40335: Correctly handle multi-line strings in tokenize error scenarios (GH-19619)
Co-authored-by: Guido van Rossum <gvanrossum@gmail.com>
Diffstat (limited to 'Parser/parsetok.c')
| -rw-r--r-- | Parser/parsetok.c | 41 |
1 files changed, 22 insertions, 19 deletions
diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 37ca65c..1ecb2c4 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -251,25 +251,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, const char *line_start; type = PyTokenizer_Get(tok, &a, &b); - if (type == ERRORTOKEN) { - err_ret->error = tok->done; - break; - } - if (type == ENDMARKER && started) { - type = NEWLINE; /* Add an extra newline */ - started = 0; - /* Add the right number of dedent tokens, - except if a certain flag is given -- - codeop.py uses this. */ - if (tok->indent && - !(*flags & PyPARSE_DONT_IMPLY_DEDENT)) - { - tok->pendin = -tok->indent; - tok->indent = 0; - } - } - else - started = 1; + len = (a != NULL && b != NULL) ? b - a : 0; str = (char *) PyObject_MALLOC(len + 1); if (str == NULL) { @@ -328,6 +310,27 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, continue; } + if (type == ERRORTOKEN) { + err_ret->error = tok->done; + break; + } + if (type == ENDMARKER && started) { + type = NEWLINE; /* Add an extra newline */ + started = 0; + /* Add the right number of dedent tokens, + except if a certain flag is given -- + codeop.py uses this. */ + if (tok->indent && + !(*flags & PyPARSE_DONT_IMPLY_DEDENT)) + { + tok->pendin = -tok->indent; + tok->indent = 0; + } + } + else { + started = 1; + } + if ((err_ret->error = PyParser_AddToken(ps, (int)type, str, lineno, col_offset, tok->lineno, end_col_offset, |