From abfbab6415fb029e7dca19ecc8d29a13da37bf71 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 13 Jun 2023 17:18:11 +0200 Subject: gh-105718: Fix buffer allocation in tokenizer with readline (#105728) --- Lib/test/test_tokenize.py | 10 ++++++++++ Parser/tokenizer.c | 4 ---- Parser/tokenizer.h | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index df9c9db..15f5363 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2241,6 +2241,16 @@ def"', """\ FSTRING_END '"' (2, 3) (2, 4) """) + self.check_tokenize('''\ +f"{ +a}"''', """\ + FSTRING_START 'f"' (1, 0) (1, 2) + LBRACE '{' (1, 2) (1, 3) + NAME 'a' (2, 0) (2, 1) + RBRACE '}' (2, 1) (2, 2) + FSTRING_END '"' (2, 2) (2, 3) + """) + self.check_tokenize(r'Rf"abc\ def"', """\ FSTRING_START 'Rf"' (1, 0) (1, 3) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index bf6bfd9..5d01b8e 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1106,11 +1106,7 @@ tok_readline_string(struct tok_state* tok) { tok->inp += buflen; *tok->inp = '\0'; - if (tok->start == NULL) { - tok->buf = tok->cur; - } tok->line_start = tok->cur; - Py_DECREF(line); return 1; error: diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 16e919a..cb44845 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -68,7 +68,7 @@ typedef struct _tokenizer_mode { struct tok_state { /* Input state; buf <= cur <= inp <= end */ /* NB an entire line is held in the buffer */ - char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ + char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ int fp_interactive; /* If the file descriptor is interactive */ -- cgit v0.12