bpo-40334: refactor and cleanup for the PEG generators (GH-19775)

3 files changed, 23 insertions, 70 deletions

diff --git a/Parser/pegen/parse.c b/Parser/pegen/parse.c
index b26f732..76dd6d3 100644
--- a/Parser/pegen/parse.c
+++ b/Parser/pegen/parse.c
@@ -648,7 +648,7 @@ file_rule(Parser *p)
         if (
             (a = statements_rule(p), 1)
             &&
-            (endmarker_var = _PyPegen_endmarker_token(p))
+            (endmarker_var = _PyPegen_expect_token(p, ENDMARKER))
         )
         {
             res = Module ( a , NULL , p -> arena );
@@ -712,7 +712,7 @@ eval_rule(Parser *p)
             &&
             (_loop0_1_var = _loop0_1_rule(p))
             &&
-            (endmarker_var = _PyPegen_endmarker_token(p))
+            (endmarker_var = _PyPegen_expect_token(p, ENDMARKER))
         )
         {
             res = Expression ( a , p -> arena );
@@ -846,7 +846,7 @@ statement_newline_rule(Parser *p)
         if (
             (a = compound_stmt_rule(p))
             &&
-            (newline_var = _PyPegen_newline_token(p))
+            (newline_var = _PyPegen_expect_token(p, NEWLINE))
         )
         {
             res = _PyPegen_singleton_seq ( p , a );
@@ -872,7 +872,7 @@ statement_newline_rule(Parser *p)
     { // NEWLINE
         void *newline_var;
         if (
-            (newline_var = _PyPegen_newline_token(p))
+            (newline_var = _PyPegen_expect_token(p, NEWLINE))
         )
         {
             Token *token = _PyPegen_get_last_nonnwhitespace_token(p);
@@ -895,7 +895,7 @@ statement_newline_rule(Parser *p)
     { // $
         void *endmarker_var;
         if (
-            (endmarker_var = _PyPegen_endmarker_token(p))
+            (endmarker_var = _PyPegen_expect_token(p, ENDMARKER))
         )
         {
             res = _PyPegen_interactive_exit ( p );
@@ -929,7 +929,7 @@ simple_stmt_rule(Parser *p)
             &&
             _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 13)
             &&
-            (newline_var = _PyPegen_newline_token(p))
+            (newline_var = _PyPegen_expect_token(p, NEWLINE))
         )
         {
             res = _PyPegen_singleton_seq ( p , a );
@@ -951,7 +951,7 @@ simple_stmt_rule(Parser *p)
             &&
             (opt_var = _PyPegen_expect_token(p, 13), 1)
             &&
-            (newline_var = _PyPegen_newline_token(p))
+            (newline_var = _PyPegen_expect_token(p, NEWLINE))
         )
         {
             res = a;
@@ -2684,7 +2684,7 @@ for_stmt_rule(Parser *p)
         void *literal;
         expr_ty t;
         if (
-            (is_async = _PyPegen_async_token(p), 1)
+            (is_async = _PyPegen_expect_token(p, ASYNC), 1)
             &&
             (keyword = _PyPegen_expect_token(p, 517))
             &&
@@ -2751,7 +2751,7 @@ with_stmt_rule(Parser *p)
         void *literal_1;
         void *literal_2;
         if (
-            (is_async = _PyPegen_async_token(p), 1)
+            (is_async = _PyPegen_expect_token(p, ASYNC), 1)
             &&
             (keyword = _PyPegen_expect_token(p, 519))
             &&
@@ -2790,7 +2790,7 @@ with_stmt_rule(Parser *p)
         void *keyword;
         void *literal;
         if (
-            (is_async = _PyPegen_async_token(p), 1)
+            (is_async = _PyPegen_expect_token(p, ASYNC), 1)
             &&
             (keyword = _PyPegen_expect_token(p, 519))
             &&
@@ -3263,7 +3263,7 @@ function_def_raw_rule(Parser *p)
         expr_ty n;
         void *params;
         if (
-            (is_async = _PyPegen_async_token(p), 1)
+            (is_async = _PyPegen_expect_token(p, ASYNC), 1)
             &&
             (keyword = _PyPegen_expect_token(p, 522))
             &&
@@ -4002,13 +4002,13 @@ block_rule(Parser *p)
         void *indent_var;
         void *newline_var;
         if (
-            (newline_var = _PyPegen_newline_token(p))
+            (newline_var = _PyPegen_expect_token(p, NEWLINE))
             &&
-            (indent_var = _PyPegen_indent_token(p))
+            (indent_var = _PyPegen_expect_token(p, INDENT))
             &&
             (a = statements_rule(p))
             &&
-            (dedent_var = _PyPegen_dedent_token(p))
+            (dedent_var = _PyPegen_expect_token(p, DEDENT))
         )
         {
             res = a;
@@ -6754,7 +6754,7 @@ await_primary_rule(Parser *p)
         expr_ty a;
         void *await_var;
         if (
-            (await_var = _PyPegen_await_token(p))
+            (await_var = _PyPegen_expect_token(p, AWAIT))
             &&
             (a = primary_rule(p))
         )
@@ -9919,9 +9919,9 @@ invalid_block_rule(Parser *p)
     { // NEWLINE !INDENT
         void *newline_var;
         if (
-            (newline_var = _PyPegen_newline_token(p))
+            (newline_var = _PyPegen_expect_token(p, NEWLINE))
             &&
-            _PyPegen_lookahead(0, _PyPegen_indent_token, p)
+            _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, INDENT)
         )
         {
             res = RAISE_INDENTATION_ERROR ( "expected an indented block" );
@@ -10036,7 +10036,7 @@ _loop0_1_rule(Parser *p)
     { // NEWLINE
         void *newline_var;
         while (
-            (newline_var = _PyPegen_newline_token(p))
+            (newline_var = _PyPegen_expect_token(p, NEWLINE))
         )
         {
             res = newline_var;
@@ -10273,7 +10273,7 @@ _tmp_6_rule(Parser *p)
     { // ASYNC
         void *async_var;
         if (
-            (async_var = _PyPegen_async_token(p))
+            (async_var = _PyPegen_expect_token(p, ASYNC))
         )
         {
             res = async_var;
@@ -10345,7 +10345,7 @@ _tmp_8_rule(Parser *p)
     { // ASYNC
         void *async_var;
         if (
-            (async_var = _PyPegen_async_token(p))
+            (async_var = _PyPegen_expect_token(p, ASYNC))
         )
         {
             res = async_var;
@@ -10381,7 +10381,7 @@ _tmp_9_rule(Parser *p)
     { // ASYNC
         void *async_var;
         if (
-            (async_var = _PyPegen_async_token(p))
+            (async_var = _PyPegen_expect_token(p, ASYNC))
         )
         {
             res = async_var;
@@ -15068,7 +15068,7 @@ _tmp_128_rule(Parser *p)
             &&
             (f = named_expression_rule(p))
             &&
-            (newline_var = _PyPegen_newline_token(p))
+            (newline_var = _PyPegen_expect_token(p, NEWLINE))
         )
         {
             res = f;
@@ -15257,7 +15257,7 @@ _tmp_134_rule(Parser *p)
         void *keyword_1;
         void *y;
         if (
-            (y = _PyPegen_async_token(p), 1)
+            (y = _PyPegen_expect_token(p, ASYNC), 1)
             &&
             (keyword = _PyPegen_expect_token(p, 517))
             &&
diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
index 39da270..942447b 100644
--- a/Parser/pegen/pegen.c
+++ b/Parser/pegen/pegen.c
@@ -693,16 +693,6 @@ _PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
 }
 
 int
-_PyPegen_lookahead_with_string(int positive, void *(func)(Parser *, const char *), Parser *p,
-                      const char *arg)
-{
-    int mark = p->mark;
-    void *res = func(p, arg);
-    p->mark = mark;
-    return (res != NULL) == positive;
-}
-
-int
 _PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
 {
     int mark = p->mark;
@@ -751,24 +741,6 @@ _PyPegen_get_last_nonnwhitespace_token(Parser *p)
     return token;
 }
 
-void *
-_PyPegen_async_token(Parser *p)
-{
-    return _PyPegen_expect_token(p, ASYNC);
-}
-
-void *
-_PyPegen_await_token(Parser *p)
-{
-    return _PyPegen_expect_token(p, AWAIT);
-}
-
-void *
-_PyPegen_endmarker_token(Parser *p)
-{
-    return _PyPegen_expect_token(p, ENDMARKER);
-}
-
 expr_ty
 _PyPegen_name_token(Parser *p)
 {
@@ -794,24 +766,6 @@ _PyPegen_string_token(Parser *p)
     return _PyPegen_expect_token(p, STRING);
 }
 
-void *
-_PyPegen_newline_token(Parser *p)
-{
-    return _PyPegen_expect_token(p, NEWLINE);
-}
-
-void *
-_PyPegen_indent_token(Parser *p)
-{
-    return _PyPegen_expect_token(p, INDENT);
-}
-
-void *
-_PyPegen_dedent_token(Parser *p)
-{
-    return _PyPegen_expect_token(p, DEDENT);
-}
-
 static PyObject *
 parsenumber_raw(const char *s)
 {
diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h
index 0ac9b31..99ec0f4 100644
--- a/Parser/pegen/pegen.h
+++ b/Parser/pegen/pegen.h
@@ -104,7 +104,6 @@ int _PyPegen_update_memo(Parser *p, int mark, int type, void *node);
 int _PyPegen_is_memoized(Parser *p, int type, void *pres);
 
 int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *);
-int _PyPegen_lookahead_with_string(int, void *(func)(Parser *, const char *), Parser *, const char *);
 int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int);
 int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);