Support error codes from plugins in options (#19719)

Mypy has options for enabling or disabling specific error codes. These
work fine, except that it is not possible to enable or disable error
codes from plugins, only mypy's original error codes.

The crux of the issue is that mypy validates and rejects unknown error
codes passed in the options before it loads plugins and learns about the
any error codes that might get registered.

There are many ways to solve this. This commit tries to find a pragmatic
solution where the relevant options parsing is deferred until after
plugin loading. Error code validation in the config parser, where
plugins are not loaded yet, is also skipped entirely, since the error
code options are re-validated later anyway. This means that this commit
introduces a small observable change in behavior when running with
invalid error codes specified, as shown in the test
test_config_file_error_codes_invalid.

This fixes #12987.

---------

Co-authored-by: John Belmonte <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Ivan Levkivskyi <[email protected]>

Author: 4 people

mypy/build.py

@@ -143,6 +143,10 @@ def __init__(self, manager: BuildManager, graph: Graph) -> None:
         self.errors: list[str] = []  # Filled in by build if desired
 
 
+def build_error(msg: str) -> NoReturn:
+    raise CompileError([f"mypy: error: {msg}"])
+
+
 def build(
     sources: list[BuildSource],
     options: Options,
@@ -241,6 +245,9 @@ def _build(
     errors = Errors(options, read_source=lambda path: read_py_file(path, cached_read))
     plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins)
 
+    # Validate error codes after plugins are loaded.
+    options.process_error_codes(error_callback=build_error)
+
     # Add catch-all .gitignore to cache dir if we created it
     cache_dir_existed = os.path.isdir(options.cache_dir)
 

mypy/config_parser.py

@@ -8,8 +8,6 @@
 import sys
 from io import StringIO
 
-from mypy.errorcodes import error_codes
-
 if sys.version_info >= (3, 11):
     import tomllib
 else:
@@ -87,15 +85,6 @@ def complain(x: object, additional_info: str = "") -> Never:
         complain(v)
 
 
-def validate_codes(codes: list[str]) -> list[str]:
-    invalid_codes = set(codes) - set(error_codes.keys())
-    if invalid_codes:
-        raise argparse.ArgumentTypeError(
-            f"Invalid error code(s): {', '.join(sorted(invalid_codes))}"
-        )
-    return codes
-
-
 def validate_package_allow_list(allow_list: list[str]) -> list[str]:
     for p in allow_list:
         msg = f"Invalid allow list entry: {p}"
@@ -209,8 +198,8 @@ def split_commas(value: str) -> list[str]:
         [p.strip() for p in split_commas(s)]
     ),
     "enable_incomplete_feature": lambda s: [p.strip() for p in split_commas(s)],
-    "disable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]),
-    "enable_error_code": lambda s: validate_codes([p.strip() for p in split_commas(s)]),
+    "disable_error_code": lambda s: [p.strip() for p in split_commas(s)],
+    "enable_error_code": lambda s: [p.strip() for p in split_commas(s)],
     "package_root": lambda s: [p.strip() for p in split_commas(s)],
     "cache_dir": expand_path,
     "python_executable": expand_path,
@@ -234,8 +223,8 @@ def split_commas(value: str) -> list[str]:
         "always_false": try_split,
         "untyped_calls_exclude": lambda s: validate_package_allow_list(try_split(s)),
         "enable_incomplete_feature": try_split,
-        "disable_error_code": lambda s: validate_codes(try_split(s)),
-        "enable_error_code": lambda s: validate_codes(try_split(s)),
+        "disable_error_code": lambda s: try_split(s),
+        "enable_error_code": lambda s: try_split(s),
         "package_root": try_split,
         "exclude": str_or_array_as_list,
         "packages": try_split,

mypy/main.py

@@ -1462,7 +1462,6 @@ def set_strict_flags() -> None:
     validate_package_allow_list(options.untyped_calls_exclude)
     validate_package_allow_list(options.deprecated_calls_exclude)
 
-    options.process_error_codes(error_callback=parser.error)
     options.process_incomplete_features(error_callback=parser.error, warning_callback=print)
 
     # Compute absolute path for custom typeshed (if present).

mypy/test/teststubtest.py

@@ -12,6 +12,8 @@
 from collections.abc import Iterator
 from typing import Any, Callable
 
+from pytest import raises
+
 import mypy.stubtest
 from mypy import build, nodes
 from mypy.modulefinder import BuildSource
@@ -171,7 +173,12 @@ def build_helper(source: str) -> build.BuildResult:
 
 
 def run_stubtest_with_stderr(
-    stub: str, runtime: str, options: list[str], config_file: str | None = None
+    stub: str,
+    runtime: str,
+    options: list[str],
+    config_file: str | None = None,
+    output: io.StringIO | None = None,
+    outerr: io.StringIO | None = None,
 ) -> tuple[str, str]:
     with use_tmp_dir(TEST_MODULE_NAME) as tmp_dir:
         with open("builtins.pyi", "w") as f:
@@ -188,8 +195,8 @@ def run_stubtest_with_stderr(
             with open(f"{TEST_MODULE_NAME}_config.ini", "w") as f:
                 f.write(config_file)
             options = options + ["--mypy-config-file", f"{TEST_MODULE_NAME}_config.ini"]
-        output = io.StringIO()
-        outerr = io.StringIO()
+        output = io.StringIO() if output is None else output
+        outerr = io.StringIO() if outerr is None else outerr
         with contextlib.redirect_stdout(output), contextlib.redirect_stderr(outerr):
             test_stubs(parse_options([TEST_MODULE_NAME] + options), use_builtins_fixtures=True)
     filtered_output = remove_color_code(
@@ -2888,14 +2895,20 @@ def test_config_file_error_codes_invalid(self) -> None:
         runtime = "temp = 5\n"
         stub = "temp: int\n"
         config_file = "[mypy]\ndisable_error_code = not-a-valid-name\n"
-        output, outerr = run_stubtest_with_stderr(
-            stub=stub, runtime=runtime, options=[], config_file=config_file
-        )
-        assert output == "Success: no issues found in 1 module\n"
-        assert outerr == (
-            "test_module_config.ini: [mypy]: disable_error_code: "
-            "Invalid error code(s): not-a-valid-name\n"
-        )
+        output = io.StringIO()
+        outerr = io.StringIO()
+        with raises(SystemExit):
+            run_stubtest_with_stderr(
+                stub=stub,
+                runtime=runtime,
+                options=[],
+                config_file=config_file,
+                output=output,
+                outerr=outerr,
+            )
+
+        assert output.getvalue() == "error: Invalid error code(s): not-a-valid-name\n"
+        assert outerr.getvalue() == ""
 
     def test_config_file_wrong_incomplete_feature(self) -> None:
         runtime = "x = 1\n"

test-data/unit/check-plugin-error-codes.test

@@ -0,0 +1,32 @@
+[case testCustomErrorCodeFromPluginIsTargetable]
+# flags: --config-file tmp/mypy.ini --show-error-codes
+
+def main() -> None:
+    return
+main() # E: Custom error  [custom]
+
+[file mypy.ini]
+\[mypy]
+plugins=<ROOT>/test-data/unit/plugins/custom_errorcode.py
+
+[case testCustomErrorCodeCanBeDisabled]
+# flags: --config-file tmp/mypy.ini --show-error-codes --disable-error-code=custom
+
+def main() -> None:
+    return
+main()  # no output expected when disabled
+
+[file mypy.ini]
+\[mypy]
+plugins=<ROOT>/test-data/unit/plugins/custom_errorcode.py
+
+[case testCustomErrorCodeCanBeReenabled]
+# flags: --config-file tmp/mypy.ini --show-error-codes --disable-error-code=custom --enable-error-code=custom
+
+def main() -> None:
+    return
+main() # E: Custom error  [custom]
+
+[file mypy.ini]
+\[mypy]
+plugins=<ROOT>/test-data/unit/plugins/custom_errorcode.py

test-data/unit/cmdline.test

@@ -960,8 +960,6 @@ src/foo/bar.py: note: Common resolutions include: a) adding `__init__.py` somewh
 [file test.py]
 x = 1
 [out]
-usage: mypy [-h] [-v] [-V] [more options; see below]
-            [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...]
 mypy: error: Invalid error code(s): YOLO
 == Return code: 2
 
@@ -970,8 +968,6 @@ mypy: error: Invalid error code(s): YOLO
 [file test.py]
 x = 1
 [out]
-usage: mypy [-h] [-v] [-V] [more options; see below]
-            [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...]
 mypy: error: Invalid error code(s): YOLO
 == Return code: 2
 
@@ -980,8 +976,6 @@ mypy: error: Invalid error code(s): YOLO
 [file test.py]
 x = 1
 [out]
-usage: mypy [-h] [-v] [-V] [more options; see below]
-            [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...]
 mypy: error: Invalid error code(s): YOLO, YOLO2
 == Return code: 2
 
@@ -990,8 +984,6 @@ mypy: error: Invalid error code(s): YOLO, YOLO2
 [file test.py]
 x = 1
 [out]
-usage: mypy [-h] [-v] [-V] [more options; see below]
-            [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...]
 mypy: error: Invalid error code(s): YOLO
 == Return code: 2
 
@@ -1000,8 +992,6 @@ mypy: error: Invalid error code(s): YOLO
 [file test.py]
 x = 1
 [out]
-usage: mypy [-h] [-v] [-V] [more options; see below]
-            [-m MODULE] [-p PACKAGE] [-c PROGRAM_TEXT] [files ...]
 mypy: error: Invalid error code(s): YOLO
 == Return code: 2