summaryrefslogtreecommitdiffstats
path: root/Parser/tokenizer.c
diff options
context:
space:
mode:
authorYury Selivanov <yselivanov@sprymix.com>2015-05-12 02:57:16 (GMT)
committerYury Selivanov <yselivanov@sprymix.com>2015-05-12 02:57:16 (GMT)
commit7544508f0245173bff5866aa1598c8f6cce1fc5f (patch)
treebf80850d9cd46fc811f04b8c2484fb50775c697d /Parser/tokenizer.c
parent4e6bf4b3da03b132b0698f30ee931a350585b117 (diff)
downloadcpython-7544508f0245173bff5866aa1598c8f6cce1fc5f.zip
cpython-7544508f0245173bff5866aa1598c8f6cce1fc5f.tar.gz
cpython-7544508f0245173bff5866aa1598c8f6cce1fc5f.tar.bz2
PEP 0492 -- Coroutines with async and await syntax. Issue #24017.
Diffstat (limited to 'Parser/tokenizer.c')
-rw-r--r--Parser/tokenizer.c68
1 files changed, 68 insertions, 0 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ac413a8..798758d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -103,6 +103,8 @@ const char *_PyParser_TokenNames[] = {
"ELLIPSIS",
/* This table must match the #defines in token.h! */
"OP",
+ "AWAIT",
+ "ASYNC",
"<ERRORTOKEN>",
"<N_TOKENS>"
};
@@ -124,6 +126,11 @@ tok_new(void)
tok->tabsize = TABSIZE;
tok->indent = 0;
tok->indstack[0] = 0;
+
+ tok->def = 0;
+ tok->defstack[0] = 0;
+ tok->deftypestack[0] = 0;
+
tok->atbol = 1;
tok->pendin = 0;
tok->prompt = tok->nextprompt = NULL;
@@ -1335,6 +1342,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
int c;
int blankline, nonascii;
+ int tok_len;
+ struct tok_state ahead_tok;
+ char *ahead_tok_start = NULL, *ahead_top_end = NULL;
+ int ahead_tok_kind;
+
*p_start = *p_end = NULL;
nextline:
tok->start = NULL;
@@ -1422,6 +1434,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
if (tok->pendin != 0) {
if (tok->pendin < 0) {
tok->pendin++;
+
+ while (tok->def && tok->defstack[tok->def] >= tok->indent) {
+ tok->def--;
+ }
+
return DEDENT;
}
else {
@@ -1481,6 +1498,57 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
return ERRORTOKEN;
*p_start = tok->start;
*p_end = tok->cur;
+
+ tok_len = tok->cur - tok->start;
+ if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
+
+ if (tok->def + 1 >= MAXINDENT) {
+ tok->done = E_TOODEEP;
+ tok->cur = tok->inp;
+ return ERRORTOKEN;
+ }
+
+ if (tok->def && tok->deftypestack[tok->def] == 3) {
+ tok->deftypestack[tok->def] = 2;
+ }
+ else {
+ tok->def++;
+ tok->defstack[tok->def] = tok->indent;
+ tok->deftypestack[tok->def] = 1;
+ }
+ }
+ else if (tok_len == 5) {
+ if (memcmp(tok->start, "async", 5) == 0) {
+ memcpy(&ahead_tok, tok, sizeof(ahead_tok));
+
+ ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
+ &ahead_top_end);
+
+ if (ahead_tok_kind == NAME &&
+ ahead_tok.cur - ahead_tok.start == 3 &&
+ memcmp(ahead_tok.start, "def", 3) == 0) {
+
+ tok->def++;
+ tok->defstack[tok->def] = tok->indent;
+ tok->deftypestack[tok->def] = 3;
+
+ return ASYNC;
+ }
+ else if (tok->def && tok->deftypestack[tok->def] == 2
+ && tok->defstack[tok->def] < tok->indent) {
+
+ return ASYNC;
+ }
+
+ }
+ else if (memcmp(tok->start, "await", 5) == 0
+ && tok->def && tok->deftypestack[tok->def] == 2
+ && tok->defstack[tok->def] < tok->indent) {
+
+ return AWAIT;
+ }
+ }
+
return NAME;
}