bpo-42997: Improve error message for missing : before suites (GH-24292)

* Add to the peg generator a new directive ('&&') that allows to expect
  a token and hard fail the parsing if the token is not found. This
  allows to quickly emmit syntax errors for missing tokens.

* Use the new grammar element to hard-fail if the ':' is missing before
  suites.

4 files changed, 70 insertions, 2 deletions

diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index 6af0d3f..f5ef5d8 100644
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -8,6 +8,7 @@ from pegen import grammar
 from pegen.grammar import (
     Alt,
     Cut,
+    Forced,
     Gather,
     GrammarVisitor,
     Group,
@@ -252,6 +253,24 @@ class CCallMakerVisitor(GrammarVisitor):
     def visit_NegativeLookahead(self, node: NegativeLookahead) -> FunctionCall:
         return self.lookahead_call_helper(node, 0)
 
+    def visit_Forced(self, node: Forced) -> FunctionCall:
+        call = self.generate_call(node.node)
+        if call.nodetype == NodeTypes.GENERIC_TOKEN:
+            val = ast.literal_eval(node.node.value)
+            assert val in self.exact_tokens, f"{node.value} is not a known literal"
+            type = self.exact_tokens[val]
+            return FunctionCall(
+                assigned_variable="_literal",
+                function=f"_PyPegen_expect_forced_token",
+                arguments=["p", type, f'"{val}"'],
+                nodetype=NodeTypes.GENERIC_TOKEN,
+                return_type="Token *",
+                comment=f"forced_token='{val}'",
+            )
+        else:
+            raise NotImplementedError(
+                    f"Forced tokens don't work with {call.nodetype} tokens")
+
     def visit_Opt(self, node: Opt) -> FunctionCall:
         call = self.generate_call(node.node)
         return FunctionCall(
diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py
index 332ee3c..66fd5b3 100644
--- a/Tools/peg_generator/pegen/grammar.py
+++ b/Tools/peg_generator/pegen/grammar.py
@@ -288,6 +288,23 @@ class NamedItem:
         gen.callmakervisitor.visit(self.item)
 
 
+class Forced:
+    def __init__(self, node: Plain):
+        self.node = node
+
+    def __str__(self) -> str:
+        return f"&&{self.node}"
+
+    def __iter__(self) -> Iterator[Plain]:
+        yield self.node
+
+    def nullable_visit(self, rules: Dict[str, Rule]) -> bool:
+        return True
+
+    def initial_names(self) -> AbstractSet[str]:
+        return set()
+
+
 class Lookahead:
     def __init__(self, node: Plain, sign: str):
         self.node = node
@@ -459,7 +476,7 @@ class Cut:
 
 
 Plain = Union[Leaf, Group]
-Item = Union[Plain, Opt, Repeat, Lookahead, Rhs, Cut]
+Item = Union[Plain, Opt, Repeat, Forced, Lookahead, Rhs, Cut]
 RuleName = Tuple[str, str]
 MetaTuple = Tuple[str, Optional[str]]
 MetaList = List[MetaTuple]
diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py
index 6e3bc50..fde0145 100644
--- a/Tools/peg_generator/pegen/grammar_parser.py
+++ b/Tools/peg_generator/pegen/grammar_parser.py
@@ -13,6 +13,7 @@ from ast import literal_eval
 from pegen.grammar import (
     Alt,
     Cut,
+    Forced,
     Gather,
     Group,
     Item,
@@ -402,7 +403,7 @@ class GeneratedParser(Parser):
 
     @memoize
     def named_item(self) -> Optional[NamedItem]:
-        # named_item: NAME '[' NAME '*' ']' '=' ~ item | NAME '[' NAME ']' '=' ~ item | NAME '=' ~ item | item | lookahead
+        # named_item: NAME '[' NAME '*' ']' '=' ~ item | NAME '[' NAME ']' '=' ~ item | NAME '=' ~ item | item | forced_atom | lookahead
         mark = self.mark()
         cut = False
         if (
@@ -466,6 +467,13 @@ class GeneratedParser(Parser):
         if cut: return None
         cut = False
         if (
+            (it := self.forced_atom())
+        ):
+            return NamedItem ( None , it )
+        self.reset(mark)
+        if cut: return None
+        cut = False
+        if (
             (it := self.lookahead())
         ):
             return NamedItem ( None , it )
@@ -474,6 +482,25 @@ class GeneratedParser(Parser):
         return None
 
     @memoize
+    def forced_atom(self) -> Optional[NamedItem]:
+        # forced_atom: '&' '&' ~ atom
+        mark = self.mark()
+        cut = False
+        if (
+            (literal := self.expect('&'))
+            and
+            (literal_1 := self.expect('&'))
+            and
+            (cut := True)
+            and
+            (atom := self.atom())
+        ):
+            return Forced ( atom )
+        self.reset(mark)
+        if cut: return None
+        return None
+
+    @memoize
     def lookahead(self) -> Optional[LookaheadOrCut]:
         # lookahead: '&' ~ atom | '!' ~ atom | '~'
         mark = self.mark()
diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram
index 4802f56..bb4355f 100644
--- a/Tools/peg_generator/pegen/metagrammar.gram
+++ b/Tools/peg_generator/pegen/metagrammar.gram
@@ -4,6 +4,7 @@ from ast import literal_eval
 from pegen.grammar import (
     Alt,
     Cut,
+    Forced,
     Gather,
     Group,
     Item,
@@ -87,8 +88,12 @@ named_item[NamedItem]:
     | NAME '[' type=NAME ']' '=' ~ item {NamedItem(name.string, item, type.string)}
     | NAME '=' ~ item {NamedItem(name.string, item)}
     | item {NamedItem(None, item)}
+    | it=forced_atom {NamedItem(None, it)}
     | it=lookahead {NamedItem(None, it)}
 
+forced_atom[NamedItem]:
+    | '&''&' ~ atom {Forced(atom)}
+
 lookahead[LookaheadOrCut]:
     | '&' ~ atom {PositiveLookahead(atom)}
     | '!' ~ atom {NegativeLookahead(atom)}