From 6282ef6c3f07b1effbf51d5806f92791751d362e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 1 Dec 2022 08:57:04 +0000 Subject: [3.11] gh-99891: Fix infinite recursion in the tokenizer when showing warnings (GH-99893) (GH-99896) Automerge-Triggered-By: GH:pablogsal. (cherry picked from commit 417206a05c4545bde96c2bbbea92b53e6cac0d48) Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_source_encoding.py | 12 ++++++++++++ .../2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst | 3 +++ Parser/tokenizer.c | 8 ++++++++ Parser/tokenizer.h | 1 + 4 files changed, 24 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index e357264..5fe0f31 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -161,6 +161,18 @@ class MiscSourceEncodingTest(unittest.TestCase): finally: os.unlink(TESTFN) + def test_tokenizer_fstring_warning_in_first_line(self): + source = "0b1and 2" + with open(TESTFN, "w") as fd: + fd.write("{}".format(source)) + try: + retcode, stdout, stderr = script_helper.assert_python_ok(TESTFN) + self.assertIn(b"SyntaxWarning: invalid binary litera", stderr) + self.assertEqual(stderr.count(source.encode()), 1) + finally: + os.unlink(TESTFN) + + class AbstractSourceEncodingTest: def test_default_coding(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst new file mode 100644 index 0000000..20cd361 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst @@ -0,0 +1,3 @@ +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index eda38a0..ca11c7b 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -88,6 +88,7 @@ tok_new(void) tok->async_def_nl = 0; tok->interactive_underflow = IUNDERFLOW_NORMAL; tok->str = NULL; + tok->report_warnings = 1; return tok; } @@ -1186,6 +1187,10 @@ indenterror(struct tok_state *tok) static int parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { + if (!tok->report_warnings) { + return 0; + } + PyObject *errmsg; va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES @@ -2194,6 +2199,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) return encoding; } } + // We don't want to report warnings here because it could cause infinite recursion + // if fetching the encoding shows a warning. + tok->report_warnings = 0; while (tok->lineno < 2 && tok->done == E_OK) { _PyTokenizer_Get(tok, &p_start, &p_end); } diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 0cb6651..d9a5f45 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -84,6 +84,7 @@ struct tok_state { NEWLINE token after it. */ /* How to proceed when asked for a new token in interactive mode */ enum interactive_underflow_t interactive_underflow; + int report_warnings; }; extern struct tok_state *_PyTokenizer_FromString(const char *, int); -- cgit v0.12