From 57f8f9a66d4d0b5e590f7746a58136b3b45b1336 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 24 Apr 2023 19:31:21 -0600 Subject: gh-103718: Correctly set f-string buffers in all cases (GH-103815) Turns out we always need to remember/restore fstring buffers in all of the stack of tokenizer modes, cause they might change to `TOK_REGULAR_MODE` and have newlines inside the braces (which is when we need to reallocate the buffer and restore the fstring ones). --- Parser/tokenizer.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5244ab7..a8649b8 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -371,10 +371,8 @@ remember_fstring_buffers(struct tok_state *tok) for (index = tok->tok_mode_stack_index; index >= 0; --index) { mode = &(tok->tok_mode_stack[index]); - if (mode->kind == TOK_FSTRING_MODE) { - mode->f_string_start_offset = mode->f_string_start - tok->buf; - mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf; - } + mode->f_string_start_offset = mode->f_string_start - tok->buf; + mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf; } } @@ -387,10 +385,8 @@ restore_fstring_buffers(struct tok_state *tok) for (index = tok->tok_mode_stack_index; index >= 0; --index) { mode = &(tok->tok_mode_stack[index]); - if (mode->kind == TOK_FSTRING_MODE) { - mode->f_string_start = tok->buf + mode->f_string_start_offset; - mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset; - } + mode->f_string_start = tok->buf + mode->f_string_start_offset; + mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset; } } @@ -1081,6 +1077,7 @@ tok_underflow_interactive(struct tok_state *tok) { restore_fstring_buffers(tok); } else { + remember_fstring_buffers(tok); ADVANCE_LINENO(); PyMem_Free(tok->buf); tok->buf = newtok; @@ -1088,6 +1085,7 @@ tok_underflow_interactive(struct tok_state *tok) { tok->line_start = tok->buf; tok->inp = strchr(tok->buf, '\0'); tok->end = tok->inp + 1; + restore_fstring_buffers(tok); } if (tok->done != E_OK) { if (tok->prompt != NULL) { -- cgit v0.12