Update pegen to use the latest upstream developments (GH-27586)

1 files changed, 164 insertions, 250 deletions

diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py
index 70fa5b0..6e9f7d3 100644
--- a/Tools/peg_generator/pegen/grammar_parser.py
+++ b/Tools/peg_generator/pegen/grammar_parser.py
@@ -2,7 +2,10 @@
 # @generated by pegen from ./Tools/peg_generator/pegen/metagrammar.gram
 
 import ast
-from typing import Optional, Any
+import sys
+import tokenize
+
+from typing import Any, Optional
 
 from pegen.parser import memoize, memoize_left_rec, logger, Parser
 from ast import literal_eval
@@ -35,83 +38,71 @@ from pegen.grammar import (
     StringLeaf,
 )
 
+# Keywords and soft keywords are listed at the end of the parser definition.
 class GeneratedParser(Parser):
 
     @memoize
     def start(self) -> Optional[Grammar]:
         # start: grammar $
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (grammar := self.grammar())
             and
-            (endmarker := self.expect('ENDMARKER'))
+            (_endmarker := self.expect('ENDMARKER'))
         ):
             return grammar
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def grammar(self) -> Optional[Grammar]:
         # grammar: metas rules | rules
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (metas := self.metas())
             and
             (rules := self.rules())
         ):
             return Grammar ( rules , metas )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (rules := self.rules())
         ):
-            return Grammar ( rules , [ ] )
-        self.reset(mark)
-        if cut: return None
+            return Grammar ( rules , [] )
+        self._reset(mark)
         return None
 
     @memoize
     def metas(self) -> Optional[MetaList]:
         # metas: meta metas | meta
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (meta := self.meta())
             and
             (metas := self.metas())
         ):
-            return [ meta ] + metas
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return [meta] + metas
+        self._reset(mark)
         if (
             (meta := self.meta())
         ):
-            return [ meta ]
-        self.reset(mark)
-        if cut: return None
+            return [meta]
+        self._reset(mark)
         return None
 
     @memoize
     def meta(self) -> Optional[MetaTuple]:
         # meta: "@" NAME NEWLINE | "@" NAME NAME NEWLINE | "@" NAME STRING NEWLINE
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (literal := self.expect("@"))
             and
             (name := self.name())
             and
-            (newline := self.expect('NEWLINE'))
+            (_newline := self.expect('NEWLINE'))
         ):
             return ( name . string , None )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (literal := self.expect("@"))
             and
@@ -119,12 +110,10 @@ class GeneratedParser(Parser):
             and
             (b := self.name())
             and
-            (newline := self.expect('NEWLINE'))
+            (_newline := self.expect('NEWLINE'))
         ):
             return ( a . string , b . string )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (literal := self.expect("@"))
             and
@@ -132,40 +121,34 @@ class GeneratedParser(Parser):
             and
             (string := self.string())
             and
-            (newline := self.expect('NEWLINE'))
+            (_newline := self.expect('NEWLINE'))
         ):
             return ( name . string , literal_eval ( string . string ) )
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def rules(self) -> Optional[RuleList]:
         # rules: rule rules | rule
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (rule := self.rule())
             and
             (rules := self.rules())
         ):
-            return [ rule ] + rules
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return [rule] + rules
+        self._reset(mark)
         if (
             (rule := self.rule())
         ):
-            return [ rule ]
-        self.reset(mark)
-        if cut: return None
+            return [rule]
+        self._reset(mark)
         return None
 
     @memoize
     def rule(self) -> Optional[Rule]:
         # rule: rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" alts NEWLINE
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (rulename := self.rulename())
             and
@@ -175,18 +158,16 @@ class GeneratedParser(Parser):
             and
             (alts := self.alts())
             and
-            (newline := self.expect('NEWLINE'))
+            (_newline := self.expect('NEWLINE'))
             and
-            (indent := self.expect('INDENT'))
+            (_indent := self.expect('INDENT'))
             and
             (more_alts := self.more_alts())
             and
-            (dedent := self.expect('DEDENT'))
+            (_dedent := self.expect('DEDENT'))
         ):
-            return Rule ( rulename [ 0 ] , rulename [ 1 ] , Rhs ( alts . alts + more_alts . alts ) , memo = opt )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return Rule ( rulename [0] , rulename [1] , Rhs ( alts . alts + more_alts . alts ) , memo = opt )
+        self._reset(mark)
         if (
             (rulename := self.rulename())
             and
@@ -194,18 +175,16 @@ class GeneratedParser(Parser):
             and
             (literal := self.expect(":"))
             and
-            (newline := self.expect('NEWLINE'))
+            (_newline := self.expect('NEWLINE'))
             and
-            (indent := self.expect('INDENT'))
+            (_indent := self.expect('INDENT'))
             and
             (more_alts := self.more_alts())
             and
-            (dedent := self.expect('DEDENT'))
+            (_dedent := self.expect('DEDENT'))
         ):
-            return Rule ( rulename [ 0 ] , rulename [ 1 ] , more_alts , memo = opt )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return Rule ( rulename [0] , rulename [1] , more_alts , memo = opt )
+        self._reset(mark)
         if (
             (rulename := self.rulename())
             and
@@ -215,76 +194,49 @@ class GeneratedParser(Parser):
             and
             (alts := self.alts())
             and
-            (newline := self.expect('NEWLINE'))
+            (_newline := self.expect('NEWLINE'))
         ):
-            return Rule ( rulename [ 0 ] , rulename [ 1 ] , alts , memo = opt )
-        self.reset(mark)
-        if cut: return None
+            return Rule ( rulename [0] , rulename [1] , alts , memo = opt )
+        self._reset(mark)
         return None
 
     @memoize
     def rulename(self) -> Optional[RuleName]:
-        # rulename: NAME '[' NAME '*' ']' | NAME '[' NAME ']' | NAME
-        mark = self.mark()
-        cut = False
-        if (
-            (name := self.name())
-            and
-            (literal := self.expect('['))
-            and
-            (type := self.name())
-            and
-            (literal_1 := self.expect('*'))
-            and
-            (literal_2 := self.expect(']'))
-        ):
-            return ( name . string , type . string + "*" )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        # rulename: NAME annotation | NAME
+        mark = self._mark()
         if (
             (name := self.name())
             and
-            (literal := self.expect('['))
-            and
-            (type := self.name())
-            and
-            (literal_1 := self.expect(']'))
+            (annotation := self.annotation())
         ):
-            return ( name . string , type . string )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return ( name . string , annotation )
+        self._reset(mark)
         if (
             (name := self.name())
         ):
             return ( name . string , None )
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def memoflag(self) -> Optional[str]:
-        # memoflag: '(' 'memo' ')'
-        mark = self.mark()
-        cut = False
+        # memoflag: '(' "memo" ')'
+        mark = self._mark()
         if (
             (literal := self.expect('('))
             and
-            (literal_1 := self.expect('memo'))
+            (literal_1 := self.expect("memo"))
             and
             (literal_2 := self.expect(')'))
         ):
             return "memo"
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def alts(self) -> Optional[Rhs]:
         # alts: alt "|" alts | alt
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (alt := self.alt())
             and
@@ -292,53 +244,45 @@ class GeneratedParser(Parser):
             and
             (alts := self.alts())
         ):
-            return Rhs ( [ alt ] + alts . alts )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return Rhs ( [alt] + alts . alts )
+        self._reset(mark)
         if (
             (alt := self.alt())
         ):
-            return Rhs ( [ alt ] )
-        self.reset(mark)
-        if cut: return None
+            return Rhs ( [alt] )
+        self._reset(mark)
         return None
 
     @memoize
     def more_alts(self) -> Optional[Rhs]:
         # more_alts: "|" alts NEWLINE more_alts | "|" alts NEWLINE
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (literal := self.expect("|"))
             and
             (alts := self.alts())
             and
-            (newline := self.expect('NEWLINE'))
+            (_newline := self.expect('NEWLINE'))
             and
             (more_alts := self.more_alts())
         ):
             return Rhs ( alts . alts + more_alts . alts )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (literal := self.expect("|"))
             and
             (alts := self.alts())
             and
-            (newline := self.expect('NEWLINE'))
+            (_newline := self.expect('NEWLINE'))
         ):
             return Rhs ( alts . alts )
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def alt(self) -> Optional[Alt]:
         # alt: items '$' action | items '$' | items action | items
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (items := self.items())
             and
@@ -346,101 +290,65 @@ class GeneratedParser(Parser):
             and
             (action := self.action())
         ):
-            return Alt ( items + [ NamedItem ( None , NameLeaf ( 'ENDMARKER' ) ) ] , action = action )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return Alt ( items + [NamedItem ( None , NameLeaf ( 'ENDMARKER' ) )] , action = action )
+        self._reset(mark)
         if (
             (items := self.items())
             and
             (literal := self.expect('$'))
         ):
-            return Alt ( items + [ NamedItem ( None , NameLeaf ( 'ENDMARKER' ) ) ] , action = None )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return Alt ( items + [NamedItem ( None , NameLeaf ( 'ENDMARKER' ) )] , action = None )
+        self._reset(mark)
         if (
             (items := self.items())
             and
             (action := self.action())
         ):
             return Alt ( items , action = action )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (items := self.items())
         ):
             return Alt ( items , action = None )
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def items(self) -> Optional[NamedItemList]:
         # items: named_item items | named_item
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (named_item := self.named_item())
             and
             (items := self.items())
         ):
-            return [ named_item ] + items
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return [named_item] + items
+        self._reset(mark)
         if (
             (named_item := self.named_item())
         ):
-            return [ named_item ]
-        self.reset(mark)
-        if cut: return None
+            return [named_item]
+        self._reset(mark)
         return None
 
     @memoize
     def named_item(self) -> Optional[NamedItem]:
-        # named_item: NAME '[' NAME '*' ']' '=' ~ item | NAME '[' NAME ']' '=' ~ item | NAME '=' ~ item | item | forced_atom | lookahead
-        mark = self.mark()
-        cut = False
-        if (
-            (name := self.name())
-            and
-            (literal := self.expect('['))
-            and
-            (type := self.name())
-            and
-            (literal_1 := self.expect('*'))
-            and
-            (literal_2 := self.expect(']'))
-            and
-            (literal_3 := self.expect('='))
-            and
-            (cut := True)
-            and
-            (item := self.item())
-        ):
-            return NamedItem ( name . string , item , f"{type.string}*" )
-        self.reset(mark)
-        if cut: return None
+        # named_item: NAME annotation '=' ~ item | NAME '=' ~ item | item | forced_atom | lookahead
+        mark = self._mark()
         cut = False
         if (
             (name := self.name())
             and
-            (literal := self.expect('['))
-            and
-            (type := self.name())
-            and
-            (literal_1 := self.expect(']'))
+            (annotation := self.annotation())
             and
-            (literal_2 := self.expect('='))
+            (literal := self.expect('='))
             and
             (cut := True)
             and
             (item := self.item())
         ):
-            return NamedItem ( name . string , item , type . string )
-        self.reset(mark)
+            return NamedItem ( name . string , item , annotation )
+        self._reset(mark)
         if cut: return None
         cut = False
         if (
@@ -453,35 +361,29 @@ class GeneratedParser(Parser):
             (item := self.item())
         ):
             return NamedItem ( name . string , item )
-        self.reset(mark)
+        self._reset(mark)
         if cut: return None
-        cut = False
         if (
             (item := self.item())
         ):
             return NamedItem ( None , item )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
-            (it := self.forced_atom())
+            (forced := self.forced_atom())
         ):
-            return NamedItem ( None , it )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+            return NamedItem ( None , forced )
+        self._reset(mark)
         if (
             (it := self.lookahead())
         ):
             return NamedItem ( None , it )
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
-    def forced_atom(self) -> Optional[NamedItem]:
+    def forced_atom(self) -> Optional[Forced]:
         # forced_atom: '&' '&' ~ atom
-        mark = self.mark()
+        mark = self._mark()
         cut = False
         if (
             (literal := self.expect('&'))
@@ -493,14 +395,14 @@ class GeneratedParser(Parser):
             (atom := self.atom())
         ):
             return Forced ( atom )
-        self.reset(mark)
+        self._reset(mark)
         if cut: return None
         return None
 
     @memoize
     def lookahead(self) -> Optional[LookaheadOrCut]:
         # lookahead: '&' ~ atom | '!' ~ atom | '~'
-        mark = self.mark()
+        mark = self._mark()
         cut = False
         if (
             (literal := self.expect('&'))
@@ -510,7 +412,7 @@ class GeneratedParser(Parser):
             (atom := self.atom())
         ):
             return PositiveLookahead ( atom )
-        self.reset(mark)
+        self._reset(mark)
         if cut: return None
         cut = False
         if (
@@ -521,21 +423,19 @@ class GeneratedParser(Parser):
             (atom := self.atom())
         ):
             return NegativeLookahead ( atom )
-        self.reset(mark)
+        self._reset(mark)
         if cut: return None
-        cut = False
         if (
             (literal := self.expect('~'))
         ):
             return Cut ( )
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def item(self) -> Optional[Item]:
         # item: '[' ~ alts ']' | atom '?' | atom '*' | atom '+' | atom '.' atom '+' | atom
-        mark = self.mark()
+        mark = self._mark()
         cut = False
         if (
             (literal := self.expect('['))
@@ -547,36 +447,29 @@ class GeneratedParser(Parser):
             (literal_1 := self.expect(']'))
         ):
             return Opt ( alts )
-        self.reset(mark)
+        self._reset(mark)
         if cut: return None
-        cut = False
         if (
             (atom := self.atom())
             and
             (literal := self.expect('?'))
         ):
             return Opt ( atom )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (atom := self.atom())
             and
             (literal := self.expect('*'))
         ):
             return Repeat0 ( atom )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (atom := self.atom())
             and
             (literal := self.expect('+'))
         ):
             return Repeat1 ( atom )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (sep := self.atom())
             and
@@ -587,21 +480,18 @@ class GeneratedParser(Parser):
             (literal_1 := self.expect('+'))
         ):
             return Gather ( sep , node )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (atom := self.atom())
         ):
             return atom
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def atom(self) -> Optional[Plain]:
         # atom: '(' ~ alts ')' | NAME | STRING
-        mark = self.mark()
+        mark = self._mark()
         cut = False
         if (
             (literal := self.expect('('))
@@ -613,28 +503,24 @@ class GeneratedParser(Parser):
             (literal_1 := self.expect(')'))
         ):
             return Group ( alts )
-        self.reset(mark)
+        self._reset(mark)
         if cut: return None
-        cut = False
         if (
             (name := self.name())
         ):
             return NameLeaf ( name . string )
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (string := self.string())
         ):
             return StringLeaf ( string . string )
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def action(self) -> Optional[str]:
         # action: "{" ~ target_atoms "}"
-        mark = self.mark()
+        mark = self._mark()
         cut = False
         if (
             (literal := self.expect("{"))
@@ -646,95 +532,123 @@ class GeneratedParser(Parser):
             (literal_1 := self.expect("}"))
         ):
             return target_atoms
-        self.reset(mark)
+        self._reset(mark)
+        if cut: return None
+        return None
+
+    @memoize
+    def annotation(self) -> Optional[str]:
+        # annotation: "[" ~ target_atoms "]"
+        mark = self._mark()
+        cut = False
+        if (
+            (literal := self.expect("["))
+            and
+            (cut := True)
+            and
+            (target_atoms := self.target_atoms())
+            and
+            (literal_1 := self.expect("]"))
+        ):
+            return target_atoms
+        self._reset(mark)
         if cut: return None
         return None
 
     @memoize
     def target_atoms(self) -> Optional[str]:
         # target_atoms: target_atom target_atoms | target_atom
-        mark = self.mark()
-        cut = False
+        mark = self._mark()
         if (
             (target_atom := self.target_atom())
             and
             (target_atoms := self.target_atoms())
         ):
             return target_atom + " " + target_atoms
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (target_atom := self.target_atom())
         ):
             return target_atom
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
     @memoize
     def target_atom(self) -> Optional[str]:
-        # target_atom: "{" ~ target_atoms "}" | NAME | NUMBER | STRING | "?" | ":" | !"}" OP
-        mark = self.mark()
+        # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | "?" | ":" | !"}" !"]" OP
+        mark = self._mark()
         cut = False
         if (
             (literal := self.expect("{"))
             and
             (cut := True)
             and
-            (target_atoms := self.target_atoms())
+            (atoms := self.target_atoms(),)
             and
             (literal_1 := self.expect("}"))
         ):
-            return "{" + target_atoms + "}"
-        self.reset(mark)
+            return "{" + ( atoms or "" ) + "}"
+        self._reset(mark)
         if cut: return None
         cut = False
         if (
+            (literal := self.expect("["))
+            and
+            (cut := True)
+            and
+            (atoms := self.target_atoms(),)
+            and
+            (literal_1 := self.expect("]"))
+        ):
+            return "[" + ( atoms or "" ) + "]"
+        self._reset(mark)
+        if cut: return None
+        if (
+            (name := self.name())
+            and
+            (literal := self.expect("*"))
+        ):
+            return name . string + "*"
+        self._reset(mark)
+        if (
             (name := self.name())
         ):
             return name . string
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (number := self.number())
         ):
             return number . string
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (string := self.string())
         ):
             return string . string
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (literal := self.expect("?"))
         ):
             return "?"
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             (literal := self.expect(":"))
         ):
             return ":"
-        self.reset(mark)
-        if cut: return None
-        cut = False
+        self._reset(mark)
         if (
             self.negative_lookahead(self.expect, "}")
             and
+            self.negative_lookahead(self.expect, "]")
+            and
             (op := self.op())
         ):
             return op . string
-        self.reset(mark)
-        if cut: return None
+        self._reset(mark)
         return None
 
+    KEYWORDS = ()
+    SOFT_KEYWORDS = ('memo',)
+
 
 if __name__ == '__main__':
     from pegen.parser import simple_parser_main