bpo-36878: Allow extra text after `# type: ignore` comments (GH-13238)

In the parser, when using the type_comments=True option, recognize
a TYPE_IGNORE as anything containing `# type: ignore` followed by
a non-alphanumeric character. This is to allow ignores such as
`# type: ignore[E1000]`.

3 files changed, 17 insertions, 9 deletions

diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py
index 69a48a1..b431890 100644
--- a/Lib/test/test_type_comments.py
+++ b/Lib/test/test_type_comments.py
@@ -76,6 +76,12 @@ def foo():
 
 def bar():
     x = 1  # type: ignore
+
+def baz():
+    pass  # type: ignore[excuse]
+    pass  # type: ignore=excuse
+    pass  # type: ignore [excuse]
+    x = 1  # type: ignore whatever
 """
 
 # Test for long-form type-comments in arguments.  A test function
@@ -266,7 +272,7 @@ class TypeCommentTests(unittest.TestCase):
 
     def test_ignores(self):
         for tree in self.parse_all(ignores):
-            self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5])
+            self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5, 8, 9, 10, 11])
         tree = self.classic_parse(ignores)
         self.assertEqual(tree.type_ignores, [])
 
@@ -318,6 +324,7 @@ class TypeCommentTests(unittest.TestCase):
         check_both_ways("while True:\n  continue  # type: int\n")
         check_both_ways("try:  # type: int\n  pass\nfinally:\n  pass\n")
         check_both_ways("try:\n  pass\nfinally:  # type: int\n  pass\n")
+        check_both_ways("pass  # type: ignorewhatever\n")
 
     def test_func_type_input(self):
 
diff --git a/Misc/NEWS.d/next/Library/2019-05-10-22-00-06.bpo-36878.iigeqk.rst b/Misc/NEWS.d/next/Library/2019-05-10-22-00-06.bpo-36878.iigeqk.rst
new file mode 100644
index 0000000..20b44dc
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-10-22-00-06.bpo-36878.iigeqk.rst
@@ -0,0 +1,4 @@
+When using `type_comments=True` in `ast.parse`, treat `# type: ignore` followed by

+a non-alphanumeric character and then arbitrary text as a type ignore, instead of

+requiring nothing but whitespace or another comment. This is to permit formations

+such as `# type: ignore[E1000]`.
\ No newline at end of file
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index e8068f2..5dc2ae6 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1272,14 +1272,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
 
                 type_start = p;
 
-                is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
-                p += 6;
-                while (is_type_ignore && p < tok->cur) {
-                    if (*p == '#')
-                        break;
-                    is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
-                    p++;
-                }
+                /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
+                 * or anything non-alphanumeric. */
+                is_type_ignore = (
+                    tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0
+                    && !(tok->cur > p + 6 && isalnum(p[6])));
 
                 if (is_type_ignore) {
                     /* If this type ignore is the only thing on the line, consume the newline also. */