bpo-36878: Allow extra text after # type: ignore comments (GH-13238)

In the parser, when using the type_comments=True option, recognize
a TYPE_IGNORE as anything containing `# type: ignore` followed by
a non-alphanumeric character. This is to allow ignores such as
`# type: ignore[E1000]`.

Author: msullivan

Lib/test/test_type_comments.py

@@ -76,6 +76,12 @@ def foo():
 
 def bar():
     x = 1  # type: ignore
+
+def baz():
+    pass  # type: ignore[excuse]
+    pass  # type: ignore=excuse
+    pass  # type: ignore [excuse]
+    x = 1  # type: ignore whatever
 """
 
 # Test for long-form type-comments in arguments.  A test function
@@ -266,7 +272,7 @@ def test_vardecl(self):
 
     def test_ignores(self):
         for tree in self.parse_all(ignores):
-            self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5])
+            self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5, 8, 9, 10, 11])
         tree = self.classic_parse(ignores)
         self.assertEqual(tree.type_ignores, [])
 
@@ -318,6 +324,7 @@ def check_both_ways(source):
         check_both_ways("while True:\n  continue  # type: int\n")
         check_both_ways("try:  # type: int\n  pass\nfinally:\n  pass\n")
         check_both_ways("try:\n  pass\nfinally:  # type: int\n  pass\n")
+        check_both_ways("pass  # type: ignorewhatever\n")
 
     def test_func_type_input(self):
 

Misc/NEWS.d/next/Library/2019-05-10-22-00-06.bpo-36878.iigeqk.rst

@@ -0,0 +1,4 @@
+When using `type_comments=True` in `ast.parse`, treat `# type: ignore` followed by
+a non-alphanumeric character and then arbitrary text as a type ignore, instead of
+requiring nothing but whitespace or another comment. This is to permit formations
+such as `# type: ignore[E1000]`.

Parser/tokenizer.c

@@ -1272,14 +1272,11 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
 
                 type_start = p;
 
-                is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
-                p += 6;
-                while (is_type_ignore && p < tok->cur) {
-                    if (*p == '#')
-                        break;
-                    is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
-                    p++;
-                }
+                /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
+                 * or anything non-alphanumeric. */
+                is_type_ignore = (
+                    tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0
+                    && !(tok->cur > p + 6 && isalnum(p[6])));
 
                 if (is_type_ignore) {
                     /* If this type ignore is the only thing on the line, consume the newline also. */