diff options
| author | Lysandros Nikolaou <lisandrosnik@gmail.com> | 2023-05-24 16:18:17 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-05-24 16:18:17 (GMT) |
| commit | c90a862cdcf55dc1753c6466e5fa4a467a13ae24 (patch) | |
| tree | 61b79498a0d900e2c653c88447409139079c0d53 /Parser | |
| parent | 9d457e115447b2079a1f66950d3c76cb77febf38 (diff) | |
| download | cpython-c90a862cdcf55dc1753c6466e5fa4a467a13ae24.zip cpython-c90a862cdcf55dc1753c6466e5fa4a467a13ae24.tar.gz cpython-c90a862cdcf55dc1753c6466e5fa4a467a13ae24.tar.bz2 | |
gh-104866: Tokenize should emit NEWLINE after exiting block with comment (#104870)
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/tokenizer.c | 9 |
1 files changed, 6 insertions, 3 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 472d417..1e8f785 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -2007,6 +2007,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t tok->atbol = 1; if (blankline || tok->level > 0) { if (tok->tok_extra_tokens) { + if (tok->comment_newline) { + tok->comment_newline = 0; + } p_start = tok->start; p_end = tok->cur; return MAKE_TOKEN(NL); @@ -2015,9 +2018,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } if (tok->comment_newline && tok->tok_extra_tokens) { tok->comment_newline = 0; - p_start = tok->start; - p_end = tok->cur; - return MAKE_TOKEN(NL); + p_start = tok->start; + p_end = tok->cur; + return MAKE_TOKEN(NL); } p_start = tok->start; p_end = tok->cur - 1; /* Leave '\n' out of the string */ |