gh-125588: Teach the python PEG generator the new f-string tokens (#125589)

Signed-off-by: Pablo Galindo <pablogsal@gmail.com>

4 files changed, 51 insertions, 2 deletions

diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py
index bf31fe5..2e3a607 100644
--- a/Tools/peg_generator/pegen/grammar_parser.py
+++ b/Tools/peg_generator/pegen/grammar_parser.py
@@ -575,7 +575,7 @@ class GeneratedParser(Parser):
 
     @memoize
     def target_atom(self) -> Optional[str]:
-        # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | "?" | ":" | !"}" !"]" OP
+        # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | FSTRING_START | FSTRING_MIDDLE | FSTRING_END | "?" | ":" | !"}" !"]" OP
         mark = self._mark()
         cut = False
         if (
@@ -626,6 +626,21 @@ class GeneratedParser(Parser):
             return string . string
         self._reset(mark)
         if (
+            (fstring_start := self.fstring_start())
+        ):
+            return fstring_start . string
+        self._reset(mark)
+        if (
+            (fstring_middle := self.fstring_middle())
+        ):
+            return fstring_middle . string
+        self._reset(mark)
+        if (
+            (fstring_end := self.fstring_end())
+        ):
+            return fstring_end . string
+        self._reset(mark)
+        if (
             (literal := self.expect("?"))
         ):
             return "?"
diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram
index f22c334..f484c47 100644
--- a/Tools/peg_generator/pegen/metagrammar.gram
+++ b/Tools/peg_generator/pegen/metagrammar.gram
@@ -126,6 +126,9 @@ target_atom[str]:
     | NAME { name.string }
     | NUMBER { number.string }
     | STRING { string.string }
+    | FSTRING_START { fstring_start.string }
+    | FSTRING_MIDDLE { fstring_middle.string }
+    | FSTRING_END { fstring_end.string }
     | "?" { "?" }
     | ":" { ":" }
     | !"}" !"]" OP { op.string }
diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py
index ed0aec9..692eb9e 100644
--- a/Tools/peg_generator/pegen/parser.py
+++ b/Tools/peg_generator/pegen/parser.py
@@ -206,6 +206,36 @@ class Parser:
         return None
 
     @memoize
+    def fstring_start(self) -> Optional[tokenize.TokenInfo]:
+        FSTRING_START = getattr(token, "FSTRING_START")
+        if not FSTRING_START:
+            return None
+        tok = self._tokenizer.peek()
+        if tok.type == FSTRING_START:
+            return self._tokenizer.getnext()
+        return None
+
+    @memoize
+    def fstring_middle(self) -> Optional[tokenize.TokenInfo]:
+        FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE")
+        if not FSTRING_MIDDLE:
+            return None
+        tok = self._tokenizer.peek()
+        if tok.type == FSTRING_MIDDLE:
+            return self._tokenizer.getnext()
+        return None
+
+    @memoize
+    def fstring_end(self) -> Optional[tokenize.TokenInfo]:
+        FSTRING_END = getattr(token, "FSTRING_END")
+        if not FSTRING_END:
+            return None
+        tok = self._tokenizer.peek()
+        if tok.type == FSTRING_END:
+            return self._tokenizer.getnext()
+        return None
+
+    @memoize
     def op(self) -> Optional[tokenize.TokenInfo]:
         tok = self._tokenizer.peek()
         if tok.type == token.OP:
diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
index 7057135..4bb2648 100644
--- a/Tools/peg_generator/pegen/python_generator.py
+++ b/Tools/peg_generator/pegen/python_generator.py
@@ -99,7 +99,8 @@ class PythonCallMakerVisitor(GrammarVisitor):
         name = node.value
         if name == "SOFT_KEYWORD":
             return "soft_keyword", "self.soft_keyword()"
-        if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"):
+        if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT",
+            "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"):
             name = name.lower()
             return name, f"self.{name}()"
         if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"):