Fix color output logic

.github/workflows/test.yml

@@ -111,14 +111,14 @@ jobs:
     env:
       TOX_SKIP_MISSING_INTERPRETERS: False
       # Rich (pip) -- Disable color for windows + pytest
-      FORCE_COLOR: ${{ !(startsWith(matrix.os, 'windows-') && startsWith(matrix.toxenv, 'py')) && 1 || 0 }}
+      #FORCE_COLOR: ${{ !(startsWith(matrix.os, 'windows-') && startsWith(matrix.toxenv, 'py')) && 1 || 0 }}
       # Tox
       PY_COLORS: 1
       # Python -- Disable argparse help colors (3.14+)
       PYTHON_COLORS: 0
       # Mypy (see https://github.com/python/mypy/issues/7771)
       TERM: xterm-color
-      MYPY_FORCE_COLOR: 1
+      #MYPY_FORCE_COLOR: 1
       MYPY_FORCE_TERMINAL_WIDTH: 200
       # Pytest
       PYTEST_ADDOPTS: --color=yes

docs/source/command_line.rst

@@ -923,9 +923,27 @@ in error messages.
     Use visually nicer output in error messages: use soft word wrap,
     show source code snippets, and show error location markers.
 
+.. option:: --color-output[=auto]
+
+    Enables colored output in error messages.
+
+    When ``--color-output=auto`` is given, uses colored output if the output
+    (both stdout and stderr) is going to a tty. This is also the default.
+
+    .. note::
+        When the environment variable ``MYPY_FORCE_COLOR`` is set to a
+        non-``0`` non-empty string, mypy always enables colored output
+        (even if ``--no-color-output`` is given).
+
+        .. Note: Here I decide not to document ``FORCE_COLOR`` as its
+           logic seems counter-intuitive from earlier conventions
+           (PR13853)
+
 .. option:: --no-color-output
 
-    This flag will disable color output in error messages, enabled by default.
+    Disables colored output in error messages.
+
+    See also note above.
 
 .. option:: --no-error-summary
 

mypy/dmypy_server.py

@@ -18,12 +18,13 @@
 import traceback
 from collections.abc import Sequence, Set as AbstractSet
 from contextlib import redirect_stderr, redirect_stdout
-from typing import Any, Callable, Final
+from typing import Any, Callable, Final, Literal, cast
 from typing_extensions import TypeAlias as _TypeAlias
 
 import mypy.build
 import mypy.errors
 import mypy.main
+from mypy import util
 from mypy.dmypy_util import WriteToConn, receive, send
 from mypy.find_sources import InvalidSourceList, create_source_list
 from mypy.fscache import FileSystemCache
@@ -199,9 +200,18 @@ def __init__(self, options: Options, status_file: str, timeout: int | None = Non
         options.local_partial_types = True
         self.status_file = status_file
 
+        # Type annotation needed for mypy (Pyright understands this)
+        use_color: bool | Literal["auto"] = (
+            True
+            if util.should_force_color()
+            else ("auto" if options.color_output == "auto" else cast(bool, options.color_output))
+        )
+
         # Since the object is created in the parent process we can check
         # the output terminal options here.
-        self.formatter = FancyFormatter(sys.stdout, sys.stderr, options.hide_error_codes)
+        self.formatter = FancyFormatter(
+            sys.stdout, sys.stderr, options.hide_error_codes, color_request=use_color
+        )
 
     def _response_metadata(self) -> dict[str, str]:
         py_version = f"{self.options.python_version[0]}_{self.options.python_version[1]}"
@@ -841,7 +851,13 @@ def pretty_messages(
         is_tty: bool = False,
         terminal_width: int | None = None,
     ) -> list[str]:
-        use_color = self.options.color_output and is_tty
+        use_color = (
+            True
+            if util.should_force_color()
+            else (
+                is_tty if self.options.color_output == "auto" else bool(self.options.color_output)
+            )
+        )
         fit_width = self.options.pretty and is_tty
         if fit_width:
             messages = self.formatter.fit_in_terminal(

mypy/main.py

@@ -11,7 +11,7 @@
 from collections.abc import Sequence
 from gettext import gettext
 from io import TextIOWrapper
-from typing import IO, TYPE_CHECKING, Any, Final, NoReturn, TextIO
+from typing import IO, TYPE_CHECKING, Any, Final, Literal, NoReturn, TextIO, cast
 
 from mypy import build, defaults, state, util
 from mypy.config_parser import (
@@ -90,8 +90,19 @@ def main(
     if clean_exit:
         options.fast_exit = False
 
+    # Type annotation needed for mypy (Pyright understands this)
+    use_color: bool | Literal["auto"] = (
+        True
+        if util.should_force_color()
+        else ("auto" if options.color_output == "auto" else cast(bool, options.color_output))
+    )
+
     formatter = util.FancyFormatter(
-        stdout, stderr, options.hide_error_codes, hide_success=bool(options.output)
+        stdout,
+        stderr,
+        options.hide_error_codes,
+        hide_success=bool(options.output),
+        color_request=use_color,
     )
 
     if options.allow_redefinition_new and not options.local_partial_types:
@@ -124,7 +135,7 @@ def main(
         install_types(formatter, options, non_interactive=options.non_interactive)
         return
 
-    res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr)
+    res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr, use_color)
 
     if options.non_interactive:
         missing_pkgs = read_types_packages_to_install(options.cache_dir, after_run=True)
@@ -133,8 +144,10 @@ def main(
             install_types(formatter, options, after_run=True, non_interactive=True)
             fscache.flush()
             print()
-            res, messages, blockers = run_build(sources, options, fscache, t0, stdout, stderr)
-        show_messages(messages, stderr, formatter, options)
+            res, messages, blockers = run_build(
+                sources, options, fscache, t0, stdout, stderr, use_color
+            )
+        show_messages(messages, stderr, formatter, options, use_color)
 
     if MEM_PROFILE:
         from mypy.memprofile import print_memory_profile
@@ -148,12 +161,12 @@ def main(
     if options.error_summary:
         if n_errors:
             summary = formatter.format_error(
-                n_errors, n_files, len(sources), blockers=blockers, use_color=options.color_output
+                n_errors, n_files, len(sources), blockers=blockers, use_color=use_color
             )
             stdout.write(summary + "\n")
         # Only notes should also output success
         elif not messages or n_notes == len(messages):
-            stdout.write(formatter.format_success(len(sources), options.color_output) + "\n")
+            stdout.write(formatter.format_success(len(sources), use_color) + "\n")
         stdout.flush()
 
     if options.install_types and not options.non_interactive:
@@ -182,9 +195,14 @@ def run_build(
     t0: float,
     stdout: TextIO,
     stderr: TextIO,
+    use_color: bool | Literal["auto"],
 ) -> tuple[build.BuildResult | None, list[str], bool]:
     formatter = util.FancyFormatter(
-        stdout, stderr, options.hide_error_codes, hide_success=bool(options.output)
+        stdout,
+        stderr,
+        options.hide_error_codes,
+        hide_success=bool(options.output),
+        color_request=use_color,
     )
 
     messages = []
@@ -200,7 +218,7 @@ def flush_errors(filename: str | None, new_messages: list[str], serious: bool) -
             # Collect messages and possibly show them later.
             return
         f = stderr if serious else stdout
-        show_messages(new_messages, f, formatter, options)
+        show_messages(new_messages, f, formatter, options, use_color)
 
     serious = False
     blockers = False
@@ -238,10 +256,16 @@ def flush_errors(filename: str | None, new_messages: list[str], serious: bool) -
 
 
 def show_messages(
-    messages: list[str], f: TextIO, formatter: util.FancyFormatter, options: Options
+    messages: list[str],
+    f: TextIO,
+    formatter: util.FancyFormatter,
+    options: Options,
+    use_color: bool | Literal["auto"],
 ) -> None:
+    if use_color == "auto":
+        use_color = formatter.default_colored
     for msg in messages:
-        if options.color_output:
+        if use_color:
             msg = formatter.colorize(msg)
         f.write(msg + "\n")
     f.flush()
@@ -462,6 +486,19 @@ def __call__(
         parser.exit()
 
 
+# Coupled with the usage in define_options
+class ColorOutputAction(argparse.Action):
+    def __call__(
+        self,
+        parser: argparse.ArgumentParser,
+        namespace: argparse.Namespace,
+        values: str | Sequence[Any] | None,
+        option_string: str | None = None,
+    ) -> None:
+        assert values in ("auto", None)
+        setattr(namespace, self.dest, True if values is None else "auto")
+
+
 def define_options(
     program: str = "mypy",
     header: str = HEADER,
@@ -993,13 +1030,22 @@ def add_invertible_flag(
         " and show error location markers",
         group=error_group,
     )
-    add_invertible_flag(
-        "--no-color-output",
+    # XXX Setting default doesn't seem to work unless I change
+    # the attribute in options.Options
+    error_group.add_argument(
+        "--color-output",
         dest="color_output",
-        default=True,
-        help="Do not colorize error messages",
-        group=error_group,
+        action=ColorOutputAction,
+        nargs="?",
+        choices=["auto"],
+        help="Colorize error messages (inverse: --no-color-output). "
+        "Detects if to use color when option is omitted and --no-color-output "
+        "is not given, or when --color-output=auto",
+    )
+    error_group.add_argument(
+        "--no-color-output", dest="color_output", action="store_false", help=argparse.SUPPRESS
     )
+    # error_group.set_defaults(color_output="auto")
     add_invertible_flag(
         "--no-error-summary",
         dest="error_summary",
@@ -1527,7 +1573,8 @@ def set_strict_flags() -> None:
                 reason = cache.find_module(p)
                 if reason is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS:
                     fail(
-                        f"Package '{p}' cannot be type checked due to missing py.typed marker. See https://mypy.readthedocs.io/en/stable/installed_packages.html for more details",
+                        f"Package '{p}' cannot be type checked due to missing py.typed marker. "
+                        "See https://mypy.readthedocs.io/en/stable/installed_packages.html for more details",
                         stderr,
                         options,
                     )

mypy/options.py

@@ -205,7 +205,7 @@ def __init__(self) -> None:
         self.show_error_context = False
 
         # Use nicer output (when possible).
-        self.color_output = True
+        self.color_output = "auto"
         self.error_summary = True
 
         # Assume arguments with default values of None are Optional

mypy/test/test_color_output.py

@@ -0,0 +1,57 @@
+from functools import partial
+from subprocess import run
+from typing import Any
+
+import pytest
+
+# TODO Would like help with this test, how do I make it runnable?
+
+
+def test(expect_color: bool, *args: Any, **kwargs: Any) -> None:
+    res = run(*args, capture_output=True, **kwargs)
+    if "Found" not in res.stdout:  # ??
+        pytest.fail("Command failed to complete or did not detect type error")
+    if expect_color:  # Expect color control chars
+        assert "<string>:1: error:" not in res.stdout
+        assert "\nFound" not in res.stdout
+    else:  # Expect no color control chars
+        assert "<string>:1: error:" in res.stdout
+        assert "\nFound" in res.stdout
+
+
+colored = partial(test, True)
+not_colored = partial(test, False)
+
+
+@pytest.mark.parametrize("command", ["mypy", "dmypy run --"])
+def test_color_output(command: str) -> None:
+    # Note: Though we don't check stderr, capturing it is useful
+    # because it provides traceback if mypy crashes due to exception
+    # and pytest reveals it upon failure (?)
+    not_colored(f"{command} -c \"1+'a'\"")
+    colored(f"{command} -c \"1+'a'\"", env={"MYPY_FORCE_COLOR": "1"})
+    colored(f"{command} -c \"1+'a'\" --color-output")
+    not_colored(f"{command} -c \"1+'a'\" --no-color-output")
+    colored(f"{command} -c \"1+'a'\" --no-color-output", env={"MYPY_FORCE_COLOR": "1"})  # TODO
+
+
+# TODO: Tests in the terminal (require manual testing?)
+"""
+In the terminal:
+    colored: mypy -c "1+'a'"
+    colored: mypy -c "1+'a'" --color-output
+not colored: mypy -c "1+'a'" --no-color-output
+    colored: mypy -c "1+'a'" --color-output (with MYPY_FORCE_COLOR=1)
+    colored: mypy -c "1+'a'" --no-color-output (with MYPY_FORCE_COLOR=1)
+
+To test, save this as a .bat and run in a Windows terminal (I don't know the Unix equivalent):
+
+set MYPY_FORCE_COLOR=
+mypy -c "1+'a'"
+mypy -c "1+'a'" --color-output
+mypy -c "1+'a'" --no-color-output
+set MYPY_FORCE_COLOR=1
+mypy -c "1+'a'" --color-output
+mypy -c "1+'a'" --no-color-output
+set MYPY_FORCE_COLOR=
+"""