PEP 0492 -- Coroutines with async and await syntax. Issue #24017.

1 files changed, 68 insertions, 0 deletions

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ac413a8..798758d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -103,6 +103,8 @@ const char *_PyParser_TokenNames[] = {
     "ELLIPSIS",
     /* This table must match the #defines in token.h! */
     "OP",
+    "AWAIT",
+    "ASYNC",
     "<ERRORTOKEN>",
     "<N_TOKENS>"
 };
@@ -124,6 +126,11 @@ tok_new(void)
     tok->tabsize = TABSIZE;
     tok->indent = 0;
     tok->indstack[0] = 0;
+
+    tok->def = 0;
+    tok->defstack[0] = 0;
+    tok->deftypestack[0] = 0;
+
     tok->atbol = 1;
     tok->pendin = 0;
     tok->prompt = tok->nextprompt = NULL;
@@ -1335,6 +1342,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     int c;
     int blankline, nonascii;
 
+    int tok_len;
+    struct tok_state ahead_tok;
+    char *ahead_tok_start = NULL, *ahead_top_end = NULL;
+    int ahead_tok_kind;
+
     *p_start = *p_end = NULL;
   nextline:
     tok->start = NULL;
@@ -1422,6 +1434,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     if (tok->pendin != 0) {
         if (tok->pendin < 0) {
             tok->pendin++;
+
+            while (tok->def && tok->defstack[tok->def] >= tok->indent) {
+                tok->def--;
+            }
+
             return DEDENT;
         }
         else {
@@ -1481,6 +1498,57 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             return ERRORTOKEN;
         *p_start = tok->start;
         *p_end = tok->cur;
+
+        tok_len = tok->cur - tok->start;
+        if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
+
+            if (tok->def + 1 >= MAXINDENT) {
+                tok->done = E_TOODEEP;
+                tok->cur = tok->inp;
+                return ERRORTOKEN;
+            }
+
+            if (tok->def && tok->deftypestack[tok->def] == 3) {
+                tok->deftypestack[tok->def] = 2;
+            }
+            else {
+                tok->def++;
+                tok->defstack[tok->def] = tok->indent;
+                tok->deftypestack[tok->def] = 1;
+            }
+        }
+        else if (tok_len == 5) {
+            if (memcmp(tok->start, "async", 5) == 0) {
+                memcpy(&ahead_tok, tok, sizeof(ahead_tok));
+
+                ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
+                                         &ahead_top_end);
+
+                if (ahead_tok_kind == NAME &&
+                        ahead_tok.cur - ahead_tok.start == 3 &&
+                        memcmp(ahead_tok.start, "def", 3) == 0) {
+
+                    tok->def++;
+                    tok->defstack[tok->def] = tok->indent;
+                    tok->deftypestack[tok->def] = 3;
+
+                    return ASYNC;
+                }
+                else if (tok->def && tok->deftypestack[tok->def] == 2
+                         && tok->defstack[tok->def] < tok->indent) {
+
+                    return ASYNC;
+                }
+
+            }
+            else if (memcmp(tok->start, "await", 5) == 0
+                        && tok->def && tok->deftypestack[tok->def] == 2
+                        && tok->defstack[tok->def] < tok->indent) {
+
+                return AWAIT;
+            }
+        }
+
         return NAME;
     }