From 2c4c26c4ce4bb94200ff3c9b5a0f4c75eed96f31 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 6 Sep 2023 06:01:23 +1000 Subject: gh-108469: Update ast.unparse for unescaped quote support from PEP701 [3.12] (#108553) Co-authored-by: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> --- Lib/ast.py | 31 +++++++--------------- Lib/test/test_tokenize.py | 2 +- Lib/test/test_unparse.py | 23 +++++++++++----- .../2023-09-03-04-37-52.gh-issue-108469.kusj40.rst | 3 +++ 4 files changed, 31 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst diff --git a/Lib/ast.py b/Lib/ast.py index 45b9596..17ec7ff 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1225,17 +1225,7 @@ class _Unparser(NodeVisitor): def visit_JoinedStr(self, node): self.write("f") - if self._avoid_backslashes: - with self.buffered() as buffer: - self._write_fstring_inner(node) - return self._write_str_avoiding_backslashes("".join(buffer)) - - # If we don't need to avoid backslashes globally (i.e., we only need - # to avoid them inside FormattedValues), it's cosmetically preferred - # to use escaped whitespace. That is, it's preferred to use backslashes - # for cases like: f"{x}\n". To accomplish this, we keep track of what - # in our buffer corresponds to FormattedValues and what corresponds to - # Constant parts of the f-string, and allow escapes accordingly. + fstring_parts = [] for value in node.values: with self.buffered() as buffer: @@ -1247,11 +1237,14 @@ class _Unparser(NodeVisitor): new_fstring_parts = [] quote_types = list(_ALL_QUOTES) for value, is_constant in fstring_parts: - value, quote_types = self._str_literal_helper( - value, - quote_types=quote_types, - escape_special_whitespace=is_constant, - ) + if is_constant: + value, quote_types = self._str_literal_helper( + value, + quote_types=quote_types, + escape_special_whitespace=True, + ) + elif "\n" in value: + quote_types = [q for q in quote_types if q in _MULTI_QUOTES] new_fstring_parts.append(value) value = "".join(new_fstring_parts) @@ -1273,16 +1266,12 @@ class _Unparser(NodeVisitor): def visit_FormattedValue(self, node): def unparse_inner(inner): - unparser = type(self)(_avoid_backslashes=True) + unparser = type(self)() unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) with self.delimit("{", "}"): expr = unparse_inner(node.value) - if "\\" in expr: - raise ValueError( - "Unable to avoid backslash in f-string expression part" - ) if expr.startswith("{"): # Separate pair of opening brackets as "{ {" self.write(" ") diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 7863e27..dbefee6 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1860,7 +1860,7 @@ class TestRoundtrip(TestCase): testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py")) - # TODO: Remove this once we can unparse PEP 701 syntax + # TODO: Remove this once we can untokenize PEP 701 syntax testfiles.remove(os.path.join(tempdir, "test_fstring.py")) for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'): diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index b3efb61..38c59e6 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -197,6 +197,10 @@ class UnparseTestCase(ASTTestCase): self.check_ast_roundtrip('''f"a\\r\\nb"''') self.check_ast_roundtrip('''f"\\u2028{'x'}"''') + def test_fstrings_pep701(self): + self.check_ast_roundtrip('f" something { my_dict["key"] } something else "') + self.check_ast_roundtrip('f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"') + def test_strings(self): self.check_ast_roundtrip("u'foo'") self.check_ast_roundtrip("r'foo'") @@ -378,8 +382,15 @@ class UnparseTestCase(ASTTestCase): ) ) - def test_invalid_fstring_backslash(self): - self.check_invalid(ast.FormattedValue(value=ast.Constant(value="\\\\"))) + def test_fstring_backslash(self): + # valid since Python 3.12 + self.assertEqual(ast.unparse( + ast.FormattedValue( + value=ast.Constant(value="\\\\"), + conversion=-1, + format_spec=None, + ) + ), "{'\\\\\\\\'}") def test_invalid_yield_from(self): self.check_invalid(ast.YieldFrom(value=None)) @@ -502,11 +513,11 @@ class CosmeticTestCase(ASTTestCase): self.check_src_roundtrip("class X(*args, **kwargs):\n pass") def test_fstrings(self): - self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') - self.check_src_roundtrip('''f"\\u2028{'x'}"''') + self.check_src_roundtrip("f'-{f'*{f'+{f'.{x}.'}+'}*'}-'") + self.check_src_roundtrip("f'\\u2028{'x'}'") self.check_src_roundtrip(r"f'{x}\n'") - self.check_src_roundtrip('''f''\'{"""\n"""}\\n''\'''') - self.check_src_roundtrip('''f''\'{f"""{x}\n"""}\\n''\'''') + self.check_src_roundtrip("f'{'\\n'}\\n'") + self.check_src_roundtrip("f'{f'{x}\\n'}\\n'") def test_docstrings(self): docstrings = ( diff --git a/Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst b/Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst new file mode 100644 index 0000000..ac0f682 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-03-04-37-52.gh-issue-108469.kusj40.rst @@ -0,0 +1,3 @@ +:func:`ast.unparse` now supports new :term:`f-string` syntax introduced in +Python 3.12. Note that the :term:`f-string` quotes are reselected for simplicity +under the new syntax. (Patch by Steven Sun) -- cgit v0.12