bpo-40334: Support type comments (GH-19780)

This implements full support for # type: <type> comments, # type: ignore <stuff> comments, and the func_type parsing mode for ast.parse() and compile().

Closes https://github.com/we-like-parsers/cpython/issues/95.

(For now, you need to use the master branch of mypy, since another issue unique to 3.9 had to be fixed there, and there's no mypy release yet.)

The only thing missing is `feature_version=N`, which is being tracked in https://github.com/we-like-parsers/cpython/issues/124.

1 files changed, 37 insertions, 7 deletions

diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h
index 99ec0f4..3af4bb2 100644
--- a/Parser/pegen/pegen.h
+++ b/Parser/pegen/pegen.h
@@ -43,6 +43,16 @@ typedef struct {
     int type;
 } KeywordToken;
 
+
+typedef struct {
+    struct {
+        int lineno;
+        char *comment;  // The " <tag>" in "# type: ignore <tag>"
+    } *items;
+    size_t size;
+    size_t num_items;
+} growable_comment_array;
+
 typedef struct {
     struct tok_state *tok;
     Token **tokens;
@@ -59,6 +69,7 @@ typedef struct {
     int starting_col_offset;
     int error_indicator;
     int flags;
+    growable_comment_array type_ignore_comments;
 } Parser;
 
 typedef struct {
@@ -110,13 +121,7 @@ int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);
 Token *_PyPegen_expect_token(Parser *p, int type);
 Token *_PyPegen_get_last_nonnwhitespace_token(Parser *);
 int _PyPegen_fill_token(Parser *p);
-void *_PyPegen_async_token(Parser *p);
-void *_PyPegen_await_token(Parser *p);
-void *_PyPegen_endmarker_token(Parser *p);
 expr_ty _PyPegen_name_token(Parser *p);
-void *_PyPegen_newline_token(Parser *p);
-void *_PyPegen_indent_token(Parser *p);
-void *_PyPegen_dedent_token(Parser *p);
 expr_ty _PyPegen_number_token(Parser *p);
 void *_PyPegen_string_token(Parser *p);
 const char *_PyPegen_get_expr_name(expr_ty);
@@ -153,6 +158,29 @@ CHECK_CALL_NULL_ALLOWED(Parser *p, void *result)
 #define CHECK(result) CHECK_CALL(p, result)
 #define CHECK_NULL_ALLOWED(result) CHECK_CALL_NULL_ALLOWED(p, result)
 
+PyObject *_PyPegen_new_type_comment(Parser *, char *);
+
+Py_LOCAL_INLINE(PyObject *)
+NEW_TYPE_COMMENT(Parser *p, Token *tc)
+{
+    if (tc == NULL) {
+        return NULL;
+    }
+    char *bytes = PyBytes_AsString(tc->bytes);
+    if (bytes == NULL) {
+        goto error;
+    }
+    PyObject *tco = _PyPegen_new_type_comment(p, bytes);
+    if (tco == NULL) {
+        goto error;
+    }
+    return tco;
+ error:
+    p->error_indicator = 1;  // Inline CHECK_CALL
+    return NULL;
+}
+
+arg_ty _PyPegen_add_type_comment_to_arg(Parser *, arg_ty, Token *);
 PyObject *_PyPegen_new_identifier(Parser *, char *);
 Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int *, PyArena *);
 void _PyPegen_Parser_Free(Parser *);
@@ -164,6 +192,7 @@ mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompiler
 void *_PyPegen_interactive_exit(Parser *);
 asdl_seq *_PyPegen_singleton_seq(Parser *, void *);
 asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *);
+asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *);
 asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *);
 expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty);
 int _PyPegen_seq_count_dots(asdl_seq *);
@@ -176,7 +205,7 @@ expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty);
 KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty);
 asdl_seq *_PyPegen_get_keys(Parser *, asdl_seq *);
 asdl_seq *_PyPegen_get_values(Parser *, asdl_seq *);
-NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty);
+NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *);
 SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_seq *, asdl_seq *);
 StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty);
 arguments_ty _PyPegen_make_arguments(Parser *, asdl_seq *, SlashWithDefault *,
@@ -192,6 +221,7 @@ expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *);
 asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
 void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
 int _PyPegen_check_barry_as_flufl(Parser *);
+mod_ty _PyPegen_make_module(Parser *, asdl_seq *);
 
 void *_PyPegen_parse(Parser *);