summaryrefslogtreecommitdiffstats
path: root/Parser
diff options
context:
space:
mode:
authorYury Selivanov <yselivanov@sprymix.com>2015-05-12 02:57:16 (GMT)
committerYury Selivanov <yselivanov@sprymix.com>2015-05-12 02:57:16 (GMT)
commit7544508f0245173bff5866aa1598c8f6cce1fc5f (patch)
treebf80850d9cd46fc811f04b8c2484fb50775c697d /Parser
parent4e6bf4b3da03b132b0698f30ee931a350585b117 (diff)
downloadcpython-7544508f0245173bff5866aa1598c8f6cce1fc5f.zip
cpython-7544508f0245173bff5866aa1598c8f6cce1fc5f.tar.gz
cpython-7544508f0245173bff5866aa1598c8f6cce1fc5f.tar.bz2
PEP 0492 -- Coroutines with async and await syntax. Issue #24017.
Diffstat (limited to 'Parser')
-rw-r--r--Parser/Python.asdl8
-rw-r--r--Parser/tokenizer.c68
-rw-r--r--Parser/tokenizer.h7
3 files changed, 82 insertions, 1 deletions
diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index 5476c53..cd0832d 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -10,7 +10,10 @@ module Python
| Suite(stmt* body)
stmt = FunctionDef(identifier name, arguments args,
- stmt* body, expr* decorator_list, expr? returns)
+ stmt* body, expr* decorator_list, expr? returns)
+ | AsyncFunctionDef(identifier name, arguments args,
+ stmt* body, expr* decorator_list, expr? returns)
+
| ClassDef(identifier name,
expr* bases,
keyword* keywords,
@@ -24,9 +27,11 @@ module Python
-- use 'orelse' because else is a keyword in target languages
| For(expr target, expr iter, stmt* body, stmt* orelse)
+ | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse)
| While(expr test, stmt* body, stmt* orelse)
| If(expr test, stmt* body, stmt* orelse)
| With(withitem* items, stmt* body)
+ | AsyncWith(withitem* items, stmt* body)
| Raise(expr? exc, expr? cause)
| Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
@@ -57,6 +62,7 @@ module Python
| DictComp(expr key, expr value, comprehension* generators)
| GeneratorExp(expr elt, comprehension* generators)
-- the grammar constrains where yield expressions can occur
+ | Await(expr value)
| Yield(expr? value)
| YieldFrom(expr value)
-- need sequences for compare to distinguish between
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ac413a8..798758d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -103,6 +103,8 @@ const char *_PyParser_TokenNames[] = {
"ELLIPSIS",
/* This table must match the #defines in token.h! */
"OP",
+ "AWAIT",
+ "ASYNC",
"<ERRORTOKEN>",
"<N_TOKENS>"
};
@@ -124,6 +126,11 @@ tok_new(void)
tok->tabsize = TABSIZE;
tok->indent = 0;
tok->indstack[0] = 0;
+
+ tok->def = 0;
+ tok->defstack[0] = 0;
+ tok->deftypestack[0] = 0;
+
tok->atbol = 1;
tok->pendin = 0;
tok->prompt = tok->nextprompt = NULL;
@@ -1335,6 +1342,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
int c;
int blankline, nonascii;
+ int tok_len;
+ struct tok_state ahead_tok;
+ char *ahead_tok_start = NULL, *ahead_top_end = NULL;
+ int ahead_tok_kind;
+
*p_start = *p_end = NULL;
nextline:
tok->start = NULL;
@@ -1422,6 +1434,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
if (tok->pendin != 0) {
if (tok->pendin < 0) {
tok->pendin++;
+
+ while (tok->def && tok->defstack[tok->def] >= tok->indent) {
+ tok->def--;
+ }
+
return DEDENT;
}
else {
@@ -1481,6 +1498,57 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
return ERRORTOKEN;
*p_start = tok->start;
*p_end = tok->cur;
+
+ tok_len = tok->cur - tok->start;
+ if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
+
+ if (tok->def + 1 >= MAXINDENT) {
+ tok->done = E_TOODEEP;
+ tok->cur = tok->inp;
+ return ERRORTOKEN;
+ }
+
+ if (tok->def && tok->deftypestack[tok->def] == 3) {
+ tok->deftypestack[tok->def] = 2;
+ }
+ else {
+ tok->def++;
+ tok->defstack[tok->def] = tok->indent;
+ tok->deftypestack[tok->def] = 1;
+ }
+ }
+ else if (tok_len == 5) {
+ if (memcmp(tok->start, "async", 5) == 0) {
+ memcpy(&ahead_tok, tok, sizeof(ahead_tok));
+
+ ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
+ &ahead_top_end);
+
+ if (ahead_tok_kind == NAME &&
+ ahead_tok.cur - ahead_tok.start == 3 &&
+ memcmp(ahead_tok.start, "def", 3) == 0) {
+
+ tok->def++;
+ tok->defstack[tok->def] = tok->indent;
+ tok->deftypestack[tok->def] = 3;
+
+ return ASYNC;
+ }
+ else if (tok->def && tok->deftypestack[tok->def] == 2
+ && tok->defstack[tok->def] < tok->indent) {
+
+ return ASYNC;
+ }
+
+ }
+ else if (memcmp(tok->start, "await", 5) == 0
+ && tok->def && tok->deftypestack[tok->def] == 2
+ && tok->defstack[tok->def] < tok->indent) {
+
+ return AWAIT;
+ }
+ }
+
return NAME;
}
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 1ce6eeb..3bcdad6 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -65,6 +65,13 @@ struct tok_state {
const char* enc; /* Encoding for the current str. */
const char* str;
const char* input; /* Tokenizer's newline translated copy of the string. */
+
+ int defstack[MAXINDENT]; /* stack if funcs & indents where they
+ were defined */
+ int deftypestack[MAXINDENT]; /* stack of func types
+ (0 not func; 1: "def name";
+ 2: "async def name") */
+ int def; /* Length of stack of func types */
};
extern struct tok_state *PyTokenizer_FromString(const char *, int);