diff options
| author | Pablo Galindo Salgado <Pablogsal@gmail.com> | 2023-06-09 20:39:01 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-06-09 20:39:01 (GMT) |
| commit | b047fa5e56ba725362c64ca3d6fccbdcf51d0cab (patch) | |
| tree | 7059182f1b40011c8051fc53a5cdb7eb2f180e15 /Parser | |
| parent | 00b599ab5a76023fa0083d7cc5d3c569342a5191 (diff) | |
| download | cpython-b047fa5e56ba725362c64ca3d6fccbdcf51d0cab.zip cpython-b047fa5e56ba725362c64ca3d6fccbdcf51d0cab.tar.gz cpython-b047fa5e56ba725362c64ca3d6fccbdcf51d0cab.tar.bz2 | |
gh-105549: Tokenize separately NUMBER and NAME tokens and allow 0-prefixed literals (#105555)
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/tokenizer.c | 13 |
1 files changed, 10 insertions, 3 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index b349f59..bf6bfd9 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1600,8 +1600,12 @@ lookahead(struct tok_state *tok, const char *test) } static int -verify_end_of_number(struct tok_state *tok, int c, const char *kind) -{ +verify_end_of_number(struct tok_state *tok, int c, const char *kind) { + if (tok->tok_extra_tokens) { + // When we are parsing extra tokens, we don't want to emit warnings + // about invalid literals, because we want to be a bit more liberal. + return 1; + } /* Emit a deprecation warning only if the numeric literal is immediately * followed by one of keywords which can occur after a numeric literal * in valid code: "and", "else", "for", "if", "in", "is" and "or". @@ -1659,6 +1663,9 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) static int verify_identifier(struct tok_state *tok) { + if (tok->tok_extra_tokens) { + return 1; + } PyObject *s; if (tok->decoding_erred) return 0; @@ -2318,7 +2325,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t else if (c == 'j' || c == 'J') { goto imaginary; } - else if (nonzero) { + else if (nonzero && !tok->tok_extra_tokens) { /* Old-style octal: now disallowed. */ tok_backup(tok, c); return MAKE_TOKEN(syntaxerror_known_range( |