Add inline typing annotations; remove .pyi stubs (#193)

* Port annotations for utility modules and remove migrated stubs

* Port checker and util_stream stub annotations inline

* Port parser and google docscrape stubs to inline annotations

* Remove remaining xdoctest type stub files

* Reduce type ignores and tighten typing fallbacks

* Fix CaptureStdout destructor safety on partial init

* Fix Python 3.8/3.9 runtime unions in typing.cast

* Make pytest override fallback independent of typing_extensions

* Add return annotations for Returns-documented functions

* Annotate Args-documented parameters across xdoctest

* Add return annotations for Returns/Yields docstring functions

* Fix ty errors

* Fix test

* Refactor away more any types

* manual type fixes

* manual type fixes

* more manual type fixes

* fix bad fstring

* fix bad fstring

* More type fixes

* Fix ty checks

* Fix new mypy errors

* Removed unused six_ast_parse code

* Fix incorrect underscore refactor

* minor cleanup

* replace more typing.Any

* Fix more typing.Any locations

* wip

* Fix mypy errors

Author: Erotemic

pyproject.toml

@@ -2,9 +2,6 @@
 requires = [ "setuptools>=41.0.1",]
 build-backend = "setuptools.build_meta"
 
-[tool.mypy]
-ignore_missing_imports = true
-
 [tool.xcookie]
 tags = [ "erotemic", "github", "purepy",]
 mod_name = "xdoctest"
@@ -93,6 +90,19 @@ count = true
 quiet-level = 3
 ignore-words-list = ['wont', 'cant', 'ANS', 'doesnt', 'arent', 'ans', 'thats', 'datas', 'isnt', 'didnt', 'wasnt']
 
+
+[tool.mypy]
+ignore_missing_imports = true
+
+[tool.ty.rules]
+unused-type-ignore-comment = "ignore"
+
+[[tool.ty.overrides]]
+include = ["src/xdoctest/_tokenize.py"]
+[tool.ty.overrides.rules]
+unsupported-operator = "ignore"
+invalid-assignment = "ignore"
+
 [tool.ruff]
 line-length = 80
 target-version = "py38"
@@ -114,3 +124,5 @@ indent-style = "space"
 skip-magic-trailing-comma = false
 line-ending = "auto"
 docstring-code-format = false
+
+

src/xdoctest/__main__.py

@@ -5,8 +5,9 @@
 This should work even if the target module is unaware of xdoctest.
 """
 
-import sys
+from __future__ import annotations
 
+import sys
 
 __tests__ = """
 Ignore:
@@ -17,7 +18,7 @@
 """
 
 
-def main(argv=None):
+def main(argv: list[str] | None = None) -> int:
     """
     Args:
         argv (List[str] | None):
@@ -44,9 +45,10 @@ def main(argv=None):
 
     import argparse
     import textwrap
-    from xdoctest import utils
     from os.path import exists
 
+    from xdoctest import utils
+
     # FIXME: default values are reporting incorrectly or are missformated
     class RawDescriptionDefaultsHelpFormatter(
         argparse.RawDescriptionHelpFormatter,
@@ -86,8 +88,7 @@ class RawDescriptionDefaultsHelpFormatter(
     )
 
     # The bulk of the argparse CLI is defined in the doctest example
-    from xdoctest import doctest_example
-    from xdoctest import runner
+    from xdoctest import doctest_example, runner
 
     runner._update_argparse_cli(parser.add_argument)
     doctest_example.DoctestConfig()._update_argparse_cli(parser.add_argument)
@@ -140,27 +141,33 @@ class RawDescriptionDefaultsHelpFormatter(
         options = ''
         pyproject_fpath = 'pyproject.toml'
         if exists(pyproject_fpath):
+            toml_loader = None
             try:
                 import tomllib
             except ImportError:
                 try:
-                    import tomli as tomllib
+                    import tomli  # type: ignore[unresolved-import]
                 except ImportError:
                     pass
+                else:
+                    toml_loader = tomli
             else:
+                toml_loader = tomllib
+
+            if toml_loader is not None:
                 with open(pyproject_fpath, 'rb') as file:
-                    pyproject_settings = tomllib.load(file)
+                    pyproject_settings = toml_loader.load(file)
                 try:
                     options = pyproject_settings['tool']['xdoctest']['options']
                 except KeyError:
                     pass
         if exists('pytest.ini'):
             import configparser
 
-            parser = configparser.ConfigParser()
-            parser.read('pytest.ini')
+            config_parser = configparser.ConfigParser()
+            config_parser.read('pytest.ini')
             try:
-                options = parser.get('pytest', 'xdoctest_options')
+                options = config_parser.get('pytest', 'xdoctest_options')
             except configparser.NoOptionError:
                 pass
         ns['options'] = options

src/xdoctest/__main__.pyi

@@ -1,4 +1,3 @@
-#  type: ignore
 # Vendored from Python 3.11
 """Tokenization help for Python programs.
 
@@ -686,7 +685,7 @@ def error(message, filename=None, location=None):
 
 def _generate_tokens_from_c_tokenizer(source):
     """Tokenize a source reading Python code as unicode strings using the internal C tokenizer"""
-    import _tokenize as c_tokenizer
+    import _tokenize as c_tokenizer  # type: ignore[unresolved-import]
     for info in c_tokenizer.TokenizerIter(source):
         tok, type, lineno, end_lineno, col_off, end_col_off, line = info
         yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line)

src/xdoctest/_tokenize.py

@@ -33,6 +33,10 @@
 representation of expression-based "got-strings".
 """
 
+from __future__ import annotations
+
+import typing
+
 import re
 import difflib
 from xdoctest import utils
@@ -66,7 +70,10 @@
 
 
 def check_got_vs_want(
-    want, got_stdout, got_eval=constants.NOT_EVALED, runstate=None
+    want: str,
+    got_stdout: str,
+    got_eval: typing.Any = constants.NOT_EVALED,
+    runstate: directive.RuntimeState | None = None,
 ):
     """
     Determines to check against either got_stdout or got_eval, and then does
@@ -121,7 +128,7 @@ def check_got_vs_want(
     return flag
 
 
-def _strip_exception_details(msg):
+def _strip_exception_details(msg: str) -> str:
     """
     Args:
         msg (str):
@@ -155,7 +162,7 @@ def _strip_exception_details(msg):
     return msg[start:end]
 
 
-def extract_exc_want(want):
+def extract_exc_want(want: str) -> str | None:
     """
     Args:
         want (str): the message supplied by the user
@@ -176,7 +183,11 @@ def extract_exc_want(want):
     return exc_want
 
 
-def check_exception(exc_got, want, runstate=None):
+def check_exception(
+    exc_got: str,
+    want: str,
+    runstate: directive.RuntimeState | None = None,
+) -> bool:
     """
     Checks want against an exception
 
@@ -200,6 +211,9 @@ def check_exception(exc_got, want, runstate=None):
     # print('exc_got = {!r}'.format(exc_got))
     # print('flag = {!r}'.format(flag))
 
+    if runstate is None:
+        runstate = directive.RuntimeState()
+
     if not flag and runstate['IGNORE_EXCEPTION_DETAIL']:
         exc_got1 = _strip_exception_details(exc_got)
         exc_want1 = _strip_exception_details(exc_want)
@@ -215,7 +229,11 @@ def check_exception(exc_got, want, runstate=None):
     return flag
 
 
-def check_output(got, want, runstate=None):
+def check_output(
+    got: str,
+    want: str,
+    runstate: directive.RuntimeState | None = None,
+) -> bool:
     """
     Does the actual comparison between `got` and `want` as long as the check is
     enabled.
@@ -243,14 +261,16 @@ def check_output(got, want, runstate=None):
     return False
 
 
-def _check_match(got, want, runstate):
+def _check_match(
+    got: str, want: str, runstate: directive.RuntimeState | dict
+) -> bool:
     """
     Does the actual comparison between `got` and `want`
 
     Args:
         got (str): normalized text produced by the test
         want (str): normalized target to match against
-        runstate (xdoctest.directive.RuntimeState | None): current state
+        runstate (xdoctest.directive.RuntimeState): current state
 
     Returns:
         bool: True if got matches want
@@ -265,7 +285,7 @@ def _check_match(got, want, runstate):
     return False
 
 
-def _ellipsis_match(got, want):
+def _ellipsis_match(got: typing.Any, want: typing.Any) -> bool:
     r"""
     The ellipsis matching algorithm taken directly from standard doctest.
 
@@ -349,7 +369,11 @@ def _ellipsis_match(got, want):
     return True
 
 
-def normalize(got, want, runstate=None):
+def normalize(
+    got: str,
+    want: str,
+    runstate: directive.RuntimeState | dict[str, object] | None = None,
+) -> tuple[str, str]:
     r"""
     Normalizes the got and want string based on the runtime state.
 
@@ -452,11 +476,12 @@ def norm_repr(a, b):
 
 
 class ExtractGotReprException(AssertionError):
+    orig_ex: Exception
     """
     Exception used when we are unable to extract a string "got"
     """
 
-    def __init__(self, msg, orig_ex):
+    def __init__(self, msg: str, orig_ex: Exception) -> None:
         """
         Args:
             msg (str): The exception message
@@ -467,12 +492,14 @@ def __init__(self, msg, orig_ex):
 
 
 class GotWantException(AssertionError):
+    got: str
+    want: str
     """
     Exception used when the "got" output of a doctest differs from the expected
     "want" output.
     """
 
-    def __init__(self, msg, got, want):
+    def __init__(self, msg: str, got: str, want: str) -> None:
         """
         Args:
             msg (str): The exception message
@@ -502,7 +529,11 @@ def _do_a_fancy_diff(self, runstate=None):
 
         return False
 
-    def output_difference(self, runstate=None, colored=True):
+    def output_difference(
+        self,
+        runstate: directive.RuntimeState | None = None,
+        colored: bool = True,
+    ) -> str:
         """
         Return a string describing the differences between the expected output
         for a given example (`example`) and the actual output (`got`).
@@ -548,12 +579,14 @@ def output_difference(self, runstate=None, colored=True):
             got_lines = got.splitlines(True)
             # Use difflib to find their differences.
             if runstate['REPORT_UDIFF']:
-                diff = difflib.unified_diff(want_lines, got_lines, n=2)
-                diff = list(diff)[2:]  # strip the diff header
+                diff = list(difflib.unified_diff(want_lines, got_lines, n=2))[
+                    2:
+                ]  # strip the diff header
                 kind = 'unified diff with -expected +actual'
             elif runstate['REPORT_CDIFF']:
-                diff = difflib.context_diff(want_lines, got_lines, n=2)
-                diff = list(diff)[2:]  # strip the diff header
+                diff = list(difflib.context_diff(want_lines, got_lines, n=2))[
+                    2:
+                ]  # strip the diff header
                 kind = 'context diff with expected followed by actual'
             elif runstate['REPORT_NDIFF']:
                 # TODO: Is there a way to make Differ ignore whitespace if that
@@ -594,7 +627,9 @@ def output_difference(self, runstate=None, colored=True):
                 text = 'Expected nothing\nGot nothing\n'
         return text
 
-    def output_repr_difference(self, runstate=None):
+    def output_repr_difference(
+        self, runstate: directive.RuntimeState | None = None
+    ) -> str:
         """
         Constructs a repr difference with minimal normalization.
 
@@ -625,7 +660,7 @@ def output_repr_difference(self, runstate=None):
         return '\n'.join(lines)
 
 
-def remove_blankline_marker(text):
+def remove_blankline_marker(text: str) -> str:
     r"""
     Args:
         text (str): input text