Add config key to extend config files

mypy/config_parser.py

@@ -16,7 +16,7 @@
     import tomli as tomllib
 
 from collections.abc import Mapping, MutableMapping, Sequence
-from typing import Any, Callable, Final, TextIO, Union
+from typing import Any, Callable, Final, TextIO, Union, cast
 from typing_extensions import Never, TypeAlias
 
 from mypy import defaults
@@ -317,7 +317,7 @@ def parse_config_file(
     stdout: TextIO | None = None,
     stderr: TextIO | None = None,
 ) -> None:
-    """Parse a config file into an Options object.
+    """Parse a config file into an Options object, following config extend arguments.
 
     Errors are written to stderr but are not fatal.
 
@@ -326,36 +326,128 @@ def parse_config_file(
     stdout = stdout or sys.stdout
     stderr = stderr or sys.stderr
 
+    strict_found = False
+
+    def set_strict(value: bool) -> None:
+        nonlocal strict_found
+        strict_found = value
+
+    ret = _parse_and_extend_config_file(
+        template=options,
+        set_strict=set_strict,
+        filename=filename,
+        stdout=stdout,
+        stderr=stderr,
+        visited=set(),
+    )
+
+    if ret is None:
+        return
+
+    file_read, mypy_updates, mypy_report_dirs, module_updates = ret
+
+    if strict_found:
+        set_strict_flags()
+
+    options.config_file = file_read
+
+    for k, v in mypy_updates.items():
+        setattr(options, k, v)
+
+    options.report_dirs.update(mypy_report_dirs)
+
+    for glob, updates in module_updates.items():
+        options.per_module_options[glob] = updates
+
+
+def _merge_updates(existing: dict[str, object], new: dict[str, object]) -> None:
+    existing["disable_error_code"] = list(
+        set(
+            cast(list[str], existing.get("disable_error_code", []))
+            + cast(list[str], new.pop("disable_error_code", []))
+        )
+    )
+    existing["enable_error_code"] = list(
+        set(
+            cast(list[str], existing.get("enable_error_code", []))
+            + cast(list[str], new.pop("enable_error_code", []))
+        )
+    )
+    existing.update(new)
+
+
+def _parse_and_extend_config_file(
+    template: Options,
+    set_strict: Callable[[bool], None],
+    filename: str | None,
+    stdout: TextIO,
+    stderr: TextIO,
+    visited: set[str],
+) -> tuple[str, dict[str, object], dict[str, str], dict[str, dict[str, object]]] | None:
     ret = (
         _parse_individual_file(filename, stderr)
         if filename is not None
         else _find_config_file(stderr)
     )
     if ret is None:
-        return
+        return None
     parser, config_types, file_read = ret
 
-    options.config_file = file_read
-    os.environ["MYPY_CONFIG_FILE_DIR"] = os.path.dirname(os.path.abspath(file_read))
+    abs_file_read = os.path.abspath(file_read)
+    if abs_file_read in visited:
+        print(f"Circular extend detected: {abs_file_read}", file=stderr)
+        return None
+    visited.add(abs_file_read)
+
+    if len(visited) == 1:
+        # set it only after the first config file is visited to allow for path variable expansions
+        # when parsing below, so recursive calls for config extend references won't overwrite it
+        os.environ["MYPY_CONFIG_FILE_DIR"] = os.path.dirname(abs_file_read)
+
+    mypy_updates: dict[str, object] = {}
+    mypy_report_dirs: dict[str, str] = {}
+    module_updates: dict[str, dict[str, object]] = {}
 
     if "mypy" not in parser:
         if filename or os.path.basename(file_read) not in defaults.SHARED_CONFIG_NAMES:
             print(f"{file_read}: No [mypy] section in config file", file=stderr)
     else:
         section = parser["mypy"]
+
+        extend = section.pop("extend", None)
+        if extend:
+            parse_ret = _parse_and_extend_config_file(
+                template=template,
+                set_strict=set_strict,
+                # refer to extend relative to directory where we found current config
+                filename=os.path.relpath(
+                    os.path.normpath(
+                        os.path.join(os.path.dirname(abs_file_read), expand_path(extend))
+                    )
+                ),
+                stdout=stdout,
+                stderr=stderr,
+                visited=visited,
+            )
+
+            if parse_ret is None:
+                print(f"{extend} is not a valid path to extend from {abs_file_read}", file=stderr)
+            else:
+                _, mypy_updates, mypy_report_dirs, module_updates = parse_ret
+
         prefix = f"{file_read}: [mypy]: "
         updates, report_dirs = parse_section(
-            prefix, options, set_strict_flags, section, config_types, stderr
+            prefix, template, set_strict, section, config_types, stderr
         )
-        for k, v in updates.items():
-            setattr(options, k, v)
-        options.report_dirs.update(report_dirs)
+        # extend and overwrite existing values with new ones
+        _merge_updates(mypy_updates, updates)
+        mypy_report_dirs.update(report_dirs)
 
     for name, section in parser.items():
         if name.startswith("mypy-"):
             prefix = get_prefix(file_read, name)
             updates, report_dirs = parse_section(
-                prefix, options, set_strict_flags, section, config_types, stderr
+                prefix, template, set_strict, section, config_types, stderr
             )
             if report_dirs:
                 print(
@@ -392,7 +484,10 @@ def parse_config_file(
                         file=stderr,
                     )
                 else:
-                    options.per_module_options[glob] = updates
+                    # extend and overwrite existing values with new ones
+                    _merge_updates(module_updates.setdefault(glob, {}), updates)
+
+    return file_read, mypy_updates, mypy_report_dirs, module_updates
 
 
 def get_prefix(file_read: str, name: str) -> str:
@@ -494,7 +589,7 @@ def destructure_overrides(toml_data: dict[str, Any]) -> dict[str, Any]:
 def parse_section(
     prefix: str,
     template: Options,
-    set_strict_flags: Callable[[], None],
+    set_strict: Callable[[bool], None],
     section: Mapping[str, Any],
     config_types: dict[str, Any],
     stderr: TextIO = sys.stderr,
@@ -586,8 +681,7 @@ def parse_section(
             print(f"{prefix}{key}: {err}", file=stderr)
             continue
         if key == "strict":
-            if v:
-                set_strict_flags()
+            set_strict(v)
             continue
         results[options_key] = v
 
@@ -686,12 +780,12 @@ def parse_mypy_comments(
         stderr = StringIO()
         strict_found = False
 
-        def set_strict_flags() -> None:
+        def set_strict(value: bool) -> None:
             nonlocal strict_found
-            strict_found = True
+            strict_found = value
 
         new_sections, reports = parse_section(
-            "", template, set_strict_flags, parser["dummy"], ini_config_types, stderr=stderr
+            "", template, set_strict, parser["dummy"], ini_config_types, stderr=stderr
         )
         errors.extend((lineno, x) for x in stderr.getvalue().strip().split("\n") if x)
         if reports:

mypy/test/test_config_parser.py

@@ -1,14 +1,16 @@
 from __future__ import annotations
 
 import contextlib
+import io
 import os
 import tempfile
 import unittest
 from collections.abc import Iterator
 from pathlib import Path
 
-from mypy.config_parser import _find_config_file
+from mypy.config_parser import _find_config_file, parse_config_file
 from mypy.defaults import CONFIG_NAMES, SHARED_CONFIG_NAMES
+from mypy.options import Options
 
 
 @contextlib.contextmanager
@@ -128,3 +130,124 @@ def test_precedence_missing_section(self) -> None:
                 result = _find_config_file()
                 assert result is not None
                 assert Path(result[2]).resolve() == parent_mypy.resolve()
+
+
+class ExtendConfigFileSuite(unittest.TestCase):
+
+    def test_extend_success(self) -> None:
+        with tempfile.TemporaryDirectory() as _tmpdir:
+            tmpdir = Path(_tmpdir)
+            with chdir(tmpdir):
+                pyproject = tmpdir / "pyproject.toml"
+                write_config(
+                    pyproject,
+                    "[tool.mypy]\n"
+                    'extend = "./folder/mypy.ini"\n'
+                    "strict = true\n"
+                    "[[tool.mypy.overrides]]\n"
+                    'module = "c"\n'
+                    'enable_error_code = ["explicit-override"]\n'
+                    "disallow_untyped_defs = true",
+                )
+                folder = tmpdir / "folder"
+                folder.mkdir()
+                write_config(
+                    folder / "mypy.ini",
+                    "[mypy]\n"
+                    "strict = False\n"
+                    "ignore_missing_imports_per_module = True\n"
+                    "[mypy-c]\n"
+                    "disallow_incomplete_defs = True",
+                )
+
+                options = Options()
+                strict_option_set = False
+
+                def set_strict_flags() -> None:
+                    nonlocal strict_option_set
+                    strict_option_set = True
+
+                stdout = io.StringIO()
+                stderr = io.StringIO()
+                parse_config_file(options, set_strict_flags, None, stdout, stderr)
+
+                assert strict_option_set is True
+                assert options.ignore_missing_imports_per_module is True
+                assert options.config_file == str(pyproject.name)
+                if os.path.realpath(pyproject.parent).startswith("/private"):
+                    # MacOS has some odd symlinks for tmp folder, resolve them to get the actual values
+                    expected_path = os.path.realpath(pyproject.parent)
+                else:
+                    expected_path = str(pyproject.parent)
+                assert os.environ["MYPY_CONFIG_FILE_DIR"] == expected_path
+
+                assert options.per_module_options["c"] == {
+                    "disable_error_code": [],
+                    "enable_error_code": ["explicit-override"],
+                    "disallow_untyped_defs": True,
+                    "disallow_incomplete_defs": True,
+                }
+
+                assert stdout.getvalue() == ""
+                assert stderr.getvalue() == ""
+
+    def test_extend_cyclic(self) -> None:
+        with tempfile.TemporaryDirectory() as _tmpdir:
+            tmpdir = Path(_tmpdir)
+            with chdir(tmpdir):
+                pyproject = tmpdir / "pyproject.toml"
+                write_config(pyproject, '[tool.mypy]\nextend = "./folder/mypy.ini"\n')
+
+                folder = tmpdir / "folder"
+                folder.mkdir()
+                ini = folder / "mypy.ini"
+                write_config(ini, "[mypy]\nextend = ../pyproject.toml\n")
+
+                options = Options()
+
+                stdout = io.StringIO()
+                stderr = io.StringIO()
+                parse_config_file(options, lambda: None, None, stdout, stderr)
+
+                if os.path.realpath(pyproject).startswith("/private"):
+                    # MacOS has some odd symlinks for tmp folder, resolve them to get the actual values
+                    expected_pyproject = os.path.realpath(pyproject)
+                    expected_ini = os.path.realpath(ini)
+                else:
+                    expected_pyproject = str(pyproject)
+                    expected_ini = str(ini)
+
+                assert stdout.getvalue() == ""
+                assert stderr.getvalue() == (
+                    f"Circular extend detected: {expected_pyproject}\n"
+                    f"../pyproject.toml is not a valid path to extend from {expected_ini}\n"
+                )
+
+    def test_extend_strict_override(self) -> None:
+        with tempfile.TemporaryDirectory() as _tmpdir:
+            tmpdir = Path(_tmpdir)
+            with chdir(tmpdir):
+                pyproject = tmpdir / "pyproject.toml"
+                write_config(
+                    pyproject, '[tool.mypy]\nextend = "./folder/mypy.ini"\nstrict = True\n'
+                )
+
+                folder = tmpdir / "folder"
+                folder.mkdir()
+                ini = folder / "mypy.ini"
+                write_config(ini, "[mypy]\nstrict = false\n")
+
+                options = Options()
+
+                stdout = io.StringIO()
+                stderr = io.StringIO()
+
+                strict_called = False
+
+                def set_strict_flags() -> None:
+                    nonlocal strict_called
+                    strict_called = True
+
+                parse_config_file(options, set_strict_flags, None, stdout, stderr)
+
+                assert strict_called is False