3 files changed, 82 insertions, 1 deletions

diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index 5476c53..cd0832d 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -10,7 +10,10 @@ module Python
         | Suite(stmt* body)
 
     stmt = FunctionDef(identifier name, arguments args,
-                           stmt* body, expr* decorator_list, expr? returns)
+                       stmt* body, expr* decorator_list, expr? returns)
+          | AsyncFunctionDef(identifier name, arguments args,
+                             stmt* body, expr* decorator_list, expr? returns)
+
           | ClassDef(identifier name,
              expr* bases,
              keyword* keywords,
@@ -24,9 +27,11 @@ module Python
 
           -- use 'orelse' because else is a keyword in target languages
           | For(expr target, expr iter, stmt* body, stmt* orelse)
+          | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse)
           | While(expr test, stmt* body, stmt* orelse)
           | If(expr test, stmt* body, stmt* orelse)
           | With(withitem* items, stmt* body)
+          | AsyncWith(withitem* items, stmt* body)
 
           | Raise(expr? exc, expr? cause)
           | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
@@ -57,6 +62,7 @@ module Python
          | DictComp(expr key, expr value, comprehension* generators)
          | GeneratorExp(expr elt, comprehension* generators)
          -- the grammar constrains where yield expressions can occur
+         | Await(expr value)
          | Yield(expr? value)
          | YieldFrom(expr value)
          -- need sequences for compare to distinguish between
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ac413a8..798758d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -103,6 +103,8 @@ const char *_PyParser_TokenNames[] = {
     "ELLIPSIS",
     /* This table must match the #defines in token.h! */
     "OP",
+    "AWAIT",
+    "ASYNC",
     "<ERRORTOKEN>",
     "<N_TOKENS>"
 };
@@ -124,6 +126,11 @@ tok_new(void)
     tok->tabsize = TABSIZE;
     tok->indent = 0;
     tok->indstack[0] = 0;
+
+    tok->def = 0;
+    tok->defstack[0] = 0;
+    tok->deftypestack[0] = 0;
+
     tok->atbol = 1;
     tok->pendin = 0;
     tok->prompt = tok->nextprompt = NULL;
@@ -1335,6 +1342,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     int c;
     int blankline, nonascii;
 
+    int tok_len;
+    struct tok_state ahead_tok;
+    char *ahead_tok_start = NULL, *ahead_top_end = NULL;
+    int ahead_tok_kind;
+
     *p_start = *p_end = NULL;
   nextline:
     tok->start = NULL;
@@ -1422,6 +1434,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     if (tok->pendin != 0) {
         if (tok->pendin < 0) {
             tok->pendin++;
+
+            while (tok->def && tok->defstack[tok->def] >= tok->indent) {
+                tok->def--;
+            }
+
             return DEDENT;
         }
         else {
@@ -1481,6 +1498,57 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             return ERRORTOKEN;
         *p_start = tok->start;
         *p_end = tok->cur;
+
+        tok_len = tok->cur - tok->start;
+        if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
+
+            if (tok->def + 1 >= MAXINDENT) {
+                tok->done = E_TOODEEP;
+                tok->cur = tok->inp;
+                return ERRORTOKEN;
+            }
+
+            if (tok->def && tok->deftypestack[tok->def] == 3) {
+                tok->deftypestack[tok->def] = 2;
+            }
+            else {
+                tok->def++;
+                tok->defstack[tok->def] = tok->indent;
+                tok->deftypestack[tok->def] = 1;
+            }
+        }
+        else if (tok_len == 5) {
+            if (memcmp(tok->start, "async", 5) == 0) {
+                memcpy(&ahead_tok, tok, sizeof(ahead_tok));
+
+                ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
+                                         &ahead_top_end);
+
+                if (ahead_tok_kind == NAME &&
+                        ahead_tok.cur - ahead_tok.start == 3 &&
+                        memcmp(ahead_tok.start, "def", 3) == 0) {
+
+                    tok->def++;
+                    tok->defstack[tok->def] = tok->indent;
+                    tok->deftypestack[tok->def] = 3;
+
+                    return ASYNC;
+                }
+                else if (tok->def && tok->deftypestack[tok->def] == 2
+                         && tok->defstack[tok->def] < tok->indent) {
+
+                    return ASYNC;
+                }
+
+            }
+            else if (memcmp(tok->start, "await", 5) == 0
+                        && tok->def && tok->deftypestack[tok->def] == 2
+                        && tok->defstack[tok->def] < tok->indent) {
+
+                return AWAIT;
+            }
+        }
+
         return NAME;
     }
 
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 1ce6eeb..3bcdad6 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -65,6 +65,13 @@ struct tok_state {
     const char* enc;        /* Encoding for the current str. */
     const char* str;
     const char* input; /* Tokenizer's newline translated copy of the string. */
+
+    int defstack[MAXINDENT];     /* stack if funcs & indents where they
+                                    were defined */
+    int deftypestack[MAXINDENT]; /* stack of func types
+                                    (0 not func; 1: "def name";
+                                     2: "async def name") */
+    int def;                     /* Length of stack of func types */
 };
 
 extern struct tok_state *PyTokenizer_FromString(const char *, int);