Fix approved stub ignore, remove normpath (#18045)

Follow up to #18038

This maybe should have been two PRs

Author: hauntsaninja

mypy/build.py

@@ -2143,7 +2143,8 @@ def parse_file(self, *, temporary: bool = False) -> None:
                     raise CompileError(
                         [
                             "mypy: can't read file '{}': {}".format(
-                                self.path, os.strerror(ioerr.errno)
+                                self.path.replace(os.getcwd() + os.sep, ""),
+                                os.strerror(ioerr.errno),
                             )
                         ],
                         module_with_blocker=self.id,
@@ -2861,7 +2862,7 @@ def log_configuration(manager: BuildManager, sources: list[BuildSource]) -> None
         manager.log(f"{'Found source:':24}{source}")
 
     # Complete list of searched paths can get very long, put them under TRACE
-    for path_type, paths in manager.search_paths._asdict().items():
+    for path_type, paths in manager.search_paths.asdict().items():
         if not paths:
             manager.trace(f"No {path_type}")
             continue

mypy/modulefinder.py

@@ -13,24 +13,43 @@
 import subprocess
 import sys
 from enum import Enum, unique
-from typing import Dict, Final, List, NamedTuple, Optional, Tuple, Union
+from typing import Dict, Final, List, Optional, Tuple, Union
 from typing_extensions import TypeAlias as _TypeAlias
 
 from mypy import pyinfo
 from mypy.errors import CompileError
 from mypy.fscache import FileSystemCache
 from mypy.nodes import MypyFile
 from mypy.options import Options
-from mypy.stubinfo import approved_stub_package_exists
+from mypy.stubinfo import stub_distribution_name
 from mypy.util import os_path_join
 
 
 # Paths to be searched in find_module().
-class SearchPaths(NamedTuple):
-    python_path: tuple[str, ...]  # where user code is found
-    mypy_path: tuple[str, ...]  # from $MYPYPATH or config variable
-    package_path: tuple[str, ...]  # from get_site_packages_dirs()
-    typeshed_path: tuple[str, ...]  # paths in typeshed
+class SearchPaths:
+    def __init__(
+        self,
+        python_path: tuple[str, ...],
+        mypy_path: tuple[str, ...],
+        package_path: tuple[str, ...],
+        typeshed_path: tuple[str, ...],
+    ) -> None:
+        # where user code is found
+        self.python_path = tuple(map(os.path.abspath, python_path))
+        # from $MYPYPATH or config variable
+        self.mypy_path = tuple(map(os.path.abspath, mypy_path))
+        # from get_site_packages_dirs()
+        self.package_path = tuple(map(os.path.abspath, package_path))
+        # paths in typeshed
+        self.typeshed_path = tuple(map(os.path.abspath, typeshed_path))
+
+    def asdict(self) -> dict[str, tuple[str, ...]]:
+        return {
+            "python_path": self.python_path,
+            "mypy_path": self.mypy_path,
+            "package_path": self.package_path,
+            "typeshed_path": self.typeshed_path,
+        }
 
 
 # Package dirs are a two-tuple of path to search and whether to verify the module
@@ -239,17 +258,17 @@ def find_module_via_source_set(self, id: str) -> ModuleSearchResult | None:
         return None
 
     def find_lib_path_dirs(self, id: str, lib_path: tuple[str, ...]) -> PackageDirs:
-        """Find which elements of a lib_path have the directory a module needs to exist.
-
-        This is run for the python_path, mypy_path, and typeshed_path search paths.
-        """
+        """Find which elements of a lib_path have the directory a module needs to exist."""
         components = id.split(".")
         dir_chain = os.sep.join(components[:-1])  # e.g., 'foo/bar'
 
         dirs = []
         for pathitem in self.get_toplevel_possibilities(lib_path, components[0]):
             # e.g., '/usr/lib/python3.4/foo/bar'
-            dir = os.path.normpath(os_path_join(pathitem, dir_chain))
+            if dir_chain:
+                dir = os_path_join(pathitem, dir_chain)
+            else:
+                dir = pathitem
             if self.fscache.isdir(dir):
                 dirs.append((dir, True))
         return dirs
@@ -418,7 +437,6 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
             for package_dir in self.find_lib_path_dirs(component, self.search_paths.package_path)
         }
         for pkg_dir in self.search_paths.package_path:
-            pkg_dir = os.path.normpath(pkg_dir)
             if pkg_dir not in candidate_package_dirs:
                 continue
             stub_name = components[0] + "-stubs"
@@ -551,8 +569,22 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
         if ancestor is not None:
             return ancestor
 
-        if approved_stub_package_exists(id):
-            return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
+        approved_dist_name = stub_distribution_name(id)
+        if approved_dist_name:
+            if len(components) == 1:
+                return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
+            # If we're a missing submodule of an already installed approved stubs, we don't want to
+            # error with APPROVED_STUBS_NOT_INSTALLED, but rather want to return NOT_FOUND.
+            for i in range(1, len(components)):
+                parent_id = ".".join(components[:i])
+                if stub_distribution_name(parent_id) == approved_dist_name:
+                    break
+            else:
+                return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
+            if self.find_module(parent_id) is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED:
+                return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
+            return ModuleNotFoundReason.NOT_FOUND
+
         if found_possible_third_party_missing_type_hints:
             return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
         return ModuleNotFoundReason.NOT_FOUND
@@ -835,7 +867,6 @@ def compute_search_paths(
     return SearchPaths(
         python_path=tuple(reversed(python_path)),
         mypy_path=tuple(mypypath),
-        # package_path and typeshed_path must be normalised and absolute via os.path.abspath
         package_path=tuple(sys_path + site_packages),
         typeshed_path=tuple(lib_path),
     )

mypy/strconv.py

@@ -112,7 +112,7 @@ def visit_mypy_file(self, o: mypy.nodes.MypyFile) -> str:
         if o.path != "main":
             # Insert path. Normalize directory separators to / to unify test
             # case# output in all platforms.
-            a.insert(0, o.path.replace(os.sep, "/"))
+            a.insert(0, o.path.replace(os.getcwd() + os.sep, "").replace(os.sep, "/"))
         if o.ignored_lines:
             a.append("IgnoredLines(%s)" % ", ".join(str(line) for line in sorted(o.ignored_lines)))
         return self.dump(a, o)

mypy/stubinfo.py

@@ -6,22 +6,6 @@ def is_module_from_legacy_bundled_package(module: str) -> bool:
     return top_level in legacy_bundled_packages
 
 
-def approved_stub_package_exists(module: str) -> bool:
-    top_level = module.split(".", 1)[0]
-    if top_level in legacy_bundled_packages:
-        return True
-    if top_level in non_bundled_packages_flat:
-        return True
-    if top_level in non_bundled_packages_namespace:
-        namespace = non_bundled_packages_namespace[top_level]
-        components = module.split(".")
-        for i in range(len(components), 0, -1):
-            module = ".".join(components[:i])
-            if module in namespace:
-                return True
-    return False
-
-
 def stub_distribution_name(module: str) -> str | None:
     top_level = module.split(".", 1)[0]
 

mypy/test/testmodulefinder.py

@@ -53,12 +53,12 @@ def test__no_namespace_packages__find_a_in_pkg1(self) -> None:
         Find find pkg1/a.py for "a" with namespace_packages False.
         """
         found_module = self.fmc_nons.find_module("a")
-        expected = os.path.join(data_path, "pkg1", "a.py")
+        expected = os.path.abspath(os.path.join(data_path, "pkg1", "a.py"))
         assert_equal(expected, found_module)
 
     def test__no_namespace_packages__find_b_in_pkg2(self) -> None:
         found_module = self.fmc_ns.find_module("b")
-        expected = os.path.join(data_path, "pkg2", "b", "__init__.py")
+        expected = os.path.abspath(os.path.join(data_path, "pkg2", "b", "__init__.py"))
         assert_equal(expected, found_module)
 
     def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None:
@@ -67,39 +67,39 @@ def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None:
         the path to the first one found in mypypath.
         """
         found_module = self.fmc_ns.find_module("nsx")
-        expected = os.path.join(data_path, "nsx-pkg1", "nsx")
+        expected = os.path.abspath(os.path.join(data_path, "nsx-pkg1", "nsx"))
         assert_equal(expected, found_module)
 
     def test__find_nsx_a_init_in_pkg1(self) -> None:
         """
         Find nsx-pkg1/nsx/a/__init__.py for "nsx.a" in namespace mode.
         """
         found_module = self.fmc_ns.find_module("nsx.a")
-        expected = os.path.join(data_path, "nsx-pkg1", "nsx", "a", "__init__.py")
+        expected = os.path.abspath(os.path.join(data_path, "nsx-pkg1", "nsx", "a", "__init__.py"))
         assert_equal(expected, found_module)
 
     def test__find_nsx_b_init_in_pkg2(self) -> None:
         """
         Find nsx-pkg2/nsx/b/__init__.py for "nsx.b" in namespace mode.
         """
         found_module = self.fmc_ns.find_module("nsx.b")
-        expected = os.path.join(data_path, "nsx-pkg2", "nsx", "b", "__init__.py")
+        expected = os.path.abspath(os.path.join(data_path, "nsx-pkg2", "nsx", "b", "__init__.py"))
         assert_equal(expected, found_module)
 
     def test__find_nsx_c_c_in_pkg3(self) -> None:
         """
         Find nsx-pkg3/nsx/c/c.py for "nsx.c.c" in namespace mode.
         """
         found_module = self.fmc_ns.find_module("nsx.c.c")
-        expected = os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py")
+        expected = os.path.abspath(os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py"))
         assert_equal(expected, found_module)
 
     def test__find_nsy_a__init_pyi(self) -> None:
         """
         Prefer nsy-pkg1/a/__init__.pyi file over __init__.py.
         """
         found_module = self.fmc_ns.find_module("nsy.a")
-        expected = os.path.join(data_path, "nsy-pkg1", "nsy", "a", "__init__.pyi")
+        expected = os.path.abspath(os.path.join(data_path, "nsy-pkg1", "nsy", "a", "__init__.pyi"))
         assert_equal(expected, found_module)
 
     def test__find_nsy_b__init_py(self) -> None:
@@ -109,7 +109,7 @@ def test__find_nsy_b__init_py(self) -> None:
         a package is preferred over a module.
         """
         found_module = self.fmc_ns.find_module("nsy.b")
-        expected = os.path.join(data_path, "nsy-pkg2", "nsy", "b", "__init__.py")
+        expected = os.path.abspath(os.path.join(data_path, "nsy-pkg2", "nsy", "b", "__init__.py"))
         assert_equal(expected, found_module)
 
     def test__find_nsy_c_pyi(self) -> None:
@@ -119,17 +119,17 @@ def test__find_nsy_c_pyi(self) -> None:
         .pyi is preferred over .py.
         """
         found_module = self.fmc_ns.find_module("nsy.c")
-        expected = os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi")
+        expected = os.path.abspath(os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi"))
         assert_equal(expected, found_module)
 
     def test__find_a_in_pkg1(self) -> None:
         found_module = self.fmc_ns.find_module("a")
-        expected = os.path.join(data_path, "pkg1", "a.py")
+        expected = os.path.abspath(os.path.join(data_path, "pkg1", "a.py"))
         assert_equal(expected, found_module)
 
     def test__find_b_init_in_pkg2(self) -> None:
         found_module = self.fmc_ns.find_module("b")
-        expected = os.path.join(data_path, "pkg2", "b", "__init__.py")
+        expected = os.path.abspath(os.path.join(data_path, "pkg2", "b", "__init__.py"))
         assert_equal(expected, found_module)
 
     def test__find_d_nowhere(self) -> None:
@@ -165,7 +165,7 @@ def setUp(self) -> None:
         self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options)
 
     def path(self, *parts: str) -> str:
-        return os.path.normpath(os.path.join(self.package_dir, *parts))
+        return os.path.abspath(os.path.join(self.package_dir, *parts))
 
     def test__packages_with_ns(self) -> None:
         cases = [
@@ -214,7 +214,7 @@ def test__packages_with_ns(self) -> None:
             # A regular package with an installed set of stubs
             ("foo.bar", self.path("foo-stubs", "bar.pyi")),
             # A regular, non-site-packages module
-            ("a", os.path.join(data_path, "pkg1", "a.py")),
+            ("a", os.path.abspath(os.path.join(data_path, "pkg1", "a.py"))),
         ]
         for module, expected in cases:
             template = "Find(" + module + ") got {}; expected {}"
@@ -269,7 +269,7 @@ def test__packages_without_ns(self) -> None:
             # A regular package with an installed set of stubs
             ("foo.bar", self.path("foo-stubs", "bar.pyi")),
             # A regular, non-site-packages module
-            ("a", os.path.join(data_path, "pkg1", "a.py")),
+            ("a", os.path.abspath(os.path.join(data_path, "pkg1", "a.py"))),
         ]
         for module, expected in cases:
             template = "Find(" + module + ") got {}; expected {}"

mypy/test/teststubinfo.py

@@ -3,7 +3,6 @@
 import unittest
 
 from mypy.stubinfo import (
-    approved_stub_package_exists,
     is_module_from_legacy_bundled_package,
     legacy_bundled_packages,
     non_bundled_packages_flat,
@@ -18,17 +17,6 @@ def test_is_legacy_bundled_packages(self) -> None:
         assert is_module_from_legacy_bundled_package("pycurl")
         assert is_module_from_legacy_bundled_package("dataclasses")
 
-    def test_approved_stub_package_exists(self) -> None:
-        assert not approved_stub_package_exists("foobar_asdf")
-        assert approved_stub_package_exists("pycurl")
-        assert approved_stub_package_exists("babel")
-        assert approved_stub_package_exists("google.cloud.ndb")
-        assert approved_stub_package_exists("google.cloud.ndb.submodule")
-        assert not approved_stub_package_exists("google.cloud.unknown")
-        assert approved_stub_package_exists("google.protobuf")
-        assert approved_stub_package_exists("google.protobuf.submodule")
-        assert not approved_stub_package_exists("google")
-
     def test_stub_distribution_name(self) -> None:
         assert stub_distribution_name("foobar_asdf") is None
         assert stub_distribution_name("pycurl") == "types-pycurl"

test-data/unit/check-modules.test

@@ -3136,23 +3136,15 @@ import google.cloud.ndb  # E: Library stubs not installed for "google.cloud.ndb"
 from google.cloud import ndb
 
 [case testMissingSubmoduleOfInstalledStubPackage]
-import bleach.xyz
-from bleach.abc import fgh
+import bleach.exists
+import bleach.xyz  # E: Cannot find implementation or library stub for module named "bleach.xyz" \
+                   # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+from bleach.abc import fgh  # E: Cannot find implementation or library stub for module named "bleach.abc"
 [file bleach/__init__.pyi]
-[out]
-main:1: error: Library stubs not installed for "bleach.xyz"
-main:1: note: Hint: "python3 -m pip install types-bleach"
-main:1: note: (or run "mypy --install-types" to install all missing stub packages)
-main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
-main:2: error: Library stubs not installed for "bleach.abc"
+[file bleach/exists.pyi]
 
-[case testMissingSubmoduleOfInstalledStubPackageIgnored-xfail]
+[case testMissingSubmoduleOfInstalledStubPackageIgnored]
 # flags: --ignore-missing-imports
-
-# TODO: testMissingSubmoduleOfInstalledStubPackageIgnored was regressed in
-# https://github.com/python/mypy/pull/15347 but didn't cause failures because we don't have a
-# package path in this unit test
-
 import bleach.xyz
 from bleach.abc import fgh
 [file bleach/__init__.pyi]