diff options
| -rw-r--r-- | Include/token.h | 4 | ||||
| -rw-r--r-- | Lib/test/test_fstring.py | 2 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Core and Builtins/2020-04-14-18-54-50.bpo-40267.Q2N6Bw.rst | 1 | ||||
| -rw-r--r-- | Parser/parsetok.c | 3 | ||||
| -rwxr-xr-x | Tools/scripts/generate_token.py | 4 |
5 files changed, 13 insertions, 1 deletions
diff --git a/Include/token.h b/Include/token.h index e08708b..9b8a3aa 100644 --- a/Include/token.h +++ b/Include/token.h @@ -78,6 +78,10 @@ extern "C" { #define ISTERMINAL(x) ((x) < NT_OFFSET) #define ISNONTERMINAL(x) ((x) >= NT_OFFSET) #define ISEOF(x) ((x) == ENDMARKER) +#define ISWHITESPACE(x) ((x) == ENDMARKER || \ + (x) == NEWLINE || \ + (x) == INDENT || \ + (x) == DEDENT) PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 8fd7cf0..fe465b7 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -713,7 +713,7 @@ non-important content # lambda doesn't work without parens, because the colon # makes the parser think it's a format_spec - self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + self.assertAllRaise(SyntaxError, 'invalid syntax', ["f'{lambda x:x}'", ]) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-04-14-18-54-50.bpo-40267.Q2N6Bw.rst b/Misc/NEWS.d/next/Core and Builtins/2020-04-14-18-54-50.bpo-40267.Q2N6Bw.rst new file mode 100644 index 0000000..a778594 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-04-14-18-54-50.bpo-40267.Q2N6Bw.rst @@ -0,0 +1 @@ +Fix the tokenizer to display the correct error message, when there is a SyntaxError on the last input character and no newline follows. It used to be `unexpected EOF while parsing`, while it should be `invalid syntax`.
\ No newline at end of file diff --git a/Parser/parsetok.c b/Parser/parsetok.c index cb94721..37ca65c 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -332,6 +332,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, PyParser_AddToken(ps, (int)type, str, lineno, col_offset, tok->lineno, end_col_offset, &(err_ret->expected))) != E_OK) { + if (tok->done == E_EOF && !ISWHITESPACE(type)) { + tok->done = E_SYNTAX; + } if (err_ret->error != E_DONE) { PyObject_FREE(str); err_ret->token = type; diff --git a/Tools/scripts/generate_token.py b/Tools/scripts/generate_token.py index f2745e8..77bb5bd 100755 --- a/Tools/scripts/generate_token.py +++ b/Tools/scripts/generate_token.py @@ -69,6 +69,10 @@ extern "C" { #define ISTERMINAL(x) ((x) < NT_OFFSET) #define ISNONTERMINAL(x) ((x) >= NT_OFFSET) #define ISEOF(x) ((x) == ENDMARKER) +#define ISWHITESPACE(x) ((x) == ENDMARKER || \\ + (x) == NEWLINE || \\ + (x) == INDENT || \\ + (x) == DEDENT) PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ |