[3.6] bpo-31852: Fix segfault caused by using the async soft keyword (GH-4122)

3 files changed, 17 insertions, 0 deletions

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 10e0ad8..ef02342 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -632,6 +632,11 @@ def"', """\
     NUMBER     '1'           (1, 8) (1, 9)
     """)
 
+        self.check_tokenize("async\\", """\
+    ERRORTOKEN '\\\\'          (1, 5) (1, 6)
+    NAME       'async'       (1, 0) (1, 5)
+    """)
+
         self.check_tokenize("a = (async = 1)", """\
     NAME       'a'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst b/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst
new file mode 100644
index 0000000..d72f41b
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst
@@ -0,0 +1,2 @@
+Fix a segmentation fault caused by a combination of the async soft keyword
+and continuation lines.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ff65f2a..ab72f61 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1563,6 +1563,9 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                 /* The current token is 'async'.
                    Look ahead one token.*/
 
+                int async_def_prev = tok->async_def;
+                tok->async_def = 2;
+
                 struct tok_state ahead_tok;
                 char *ahead_tok_start = NULL, *ahead_tok_end = NULL;
                 int ahead_tok_kind;
@@ -1581,6 +1584,9 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                     tok->async_def = 1;
                     return ASYNC;
                 }
+                else{
+                    tok->async_def = async_def_prev;
+                }
             }
         }
 
@@ -1844,6 +1850,10 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     /* Line continuation */
     if (c == '\\') {
         c = tok_nextc(tok);
+        if (tok->async_def == 2) {
+            tok->done = E_SYNTAX;
+            return ERRORTOKEN;
+        }
         if (c != '\n') {
             tok->done = E_LINECONT;
             tok->cur = tok->inp;