Convert most of the collection code from py.path to pathlib

Author: bluetech

changelog/8174.trivial.rst

@@ -3,3 +3,4 @@ The following changes have been made to internal pytest types/functions:
 - The ``path`` property of ``_pytest.code.Code`` returns ``Path`` instead of ``py.path.local``.
 - The ``path`` property of ``_pytest.code.TracebackEntry`` returns ``Path`` instead of ``py.path.local``.
 - The ``_pytest.code.getfslineno()`` function returns ``Path`` instead of ``py.path.local``.
+- The ``_pytest.python.path_matches_patterns()`` function takes ``Path`` instead of ``py.path.local``.

src/_pytest/config/__init__.py

@@ -348,7 +348,7 @@ def __init__(self) -> None:
         self._conftestpath2mod: Dict[Path, types.ModuleType] = {}
         self._confcutdir: Optional[Path] = None
         self._noconftest = False
-        self._duplicatepaths: Set[py.path.local] = set()
+        self._duplicatepaths: Set[Path] = set()
 
         # plugins that were explicitly skipped with pytest.skip
         # list of (module name, skip reason)

src/_pytest/main.py

@@ -37,6 +37,7 @@
 from _pytest.outcomes import exit
 from _pytest.pathlib import absolutepath
 from _pytest.pathlib import bestrelpath
+from _pytest.pathlib import fnmatch_ex
 from _pytest.pathlib import visit
 from _pytest.reports import CollectReport
 from _pytest.reports import TestReport
@@ -353,11 +354,14 @@ def pytest_runtestloop(session: "Session") -> bool:
     return True
 
 
-def _in_venv(path: py.path.local) -> bool:
+def _in_venv(path: Path) -> bool:
     """Attempt to detect if ``path`` is the root of a Virtual Environment by
     checking for the existence of the appropriate activate script."""
-    bindir = path.join("Scripts" if sys.platform.startswith("win") else "bin")
-    if not bindir.isdir():
+    bindir = path.joinpath("Scripts" if sys.platform.startswith("win") else "bin")
+    try:
+        if not bindir.is_dir():
+            return False
+    except OSError:
         return False
     activates = (
         "activate",
@@ -367,33 +371,32 @@ def _in_venv(path: py.path.local) -> bool:
         "Activate.bat",
         "Activate.ps1",
     )
-    return any([fname.basename in activates for fname in bindir.listdir()])
+    return any(fname.name in activates for fname in bindir.iterdir())
 
 
-def pytest_ignore_collect(path: py.path.local, config: Config) -> Optional[bool]:
-    path_ = Path(path)
-    ignore_paths = config._getconftest_pathlist("collect_ignore", path=path_.parent)
+def pytest_ignore_collect(fspath: Path, config: Config) -> Optional[bool]:
+    ignore_paths = config._getconftest_pathlist("collect_ignore", path=fspath.parent)
     ignore_paths = ignore_paths or []
     excludeopt = config.getoption("ignore")
     if excludeopt:
         ignore_paths.extend(absolutepath(x) for x in excludeopt)
 
-    if path_ in ignore_paths:
+    if fspath in ignore_paths:
         return True
 
     ignore_globs = config._getconftest_pathlist(
-        "collect_ignore_glob", path=path_.parent
+        "collect_ignore_glob", path=fspath.parent
     )
     ignore_globs = ignore_globs or []
     excludeglobopt = config.getoption("ignore_glob")
     if excludeglobopt:
         ignore_globs.extend(absolutepath(x) for x in excludeglobopt)
 
-    if any(fnmatch.fnmatch(str(path), str(glob)) for glob in ignore_globs):
+    if any(fnmatch.fnmatch(str(fspath), str(glob)) for glob in ignore_globs):
         return True
 
     allow_in_venv = config.getoption("collect_in_virtualenv")
-    if not allow_in_venv and _in_venv(path):
+    if not allow_in_venv and _in_venv(fspath):
         return True
     return None
 
@@ -538,21 +541,21 @@ def _recurse(self, direntry: "os.DirEntry[str]") -> bool:
         if ihook.pytest_ignore_collect(fspath=fspath, path=path, config=self.config):
             return False
         norecursepatterns = self.config.getini("norecursedirs")
-        if any(path.check(fnmatch=pat) for pat in norecursepatterns):
+        if any(fnmatch_ex(pat, fspath) for pat in norecursepatterns):
             return False
         return True
 
     def _collectfile(
-        self, path: py.path.local, handle_dupes: bool = True
+        self, fspath: Path, handle_dupes: bool = True
     ) -> Sequence[nodes.Collector]:
-        fspath = Path(path)
+        path = py.path.local(fspath)
         assert (
-            path.isfile()
+            fspath.is_file()
         ), "{!r} is not a file (isdir={!r}, exists={!r}, islink={!r})".format(
-            path, path.isdir(), path.exists(), path.islink()
+            fspath, fspath.is_dir(), fspath.exists(), fspath.is_symlink()
         )
-        ihook = self.gethookproxy(path)
-        if not self.isinitpath(path):
+        ihook = self.gethookproxy(fspath)
+        if not self.isinitpath(fspath):
             if ihook.pytest_ignore_collect(
                 fspath=fspath, path=path, config=self.config
             ):
@@ -562,10 +565,10 @@ def _collectfile(
             keepduplicates = self.config.getoption("keepduplicates")
             if not keepduplicates:
                 duplicate_paths = self.config.pluginmanager._duplicatepaths
-                if path in duplicate_paths:
+                if fspath in duplicate_paths:
                     return ()
                 else:
-                    duplicate_paths.add(path)
+                    duplicate_paths.add(fspath)
 
         return ihook.pytest_collect_file(fspath=fspath, path=path, parent=self)  # type: ignore[no-any-return]
 
@@ -652,10 +655,8 @@ def collect(self) -> Iterator[Union[nodes.Item, nodes.Collector]]:
         from _pytest.python import Package
 
         # Keep track of any collected nodes in here, so we don't duplicate fixtures.
-        node_cache1: Dict[py.path.local, Sequence[nodes.Collector]] = {}
-        node_cache2: Dict[
-            Tuple[Type[nodes.Collector], py.path.local], nodes.Collector
-        ] = ({})
+        node_cache1: Dict[Path, Sequence[nodes.Collector]] = {}
+        node_cache2: Dict[Tuple[Type[nodes.Collector], Path], nodes.Collector] = ({})
 
         # Keep track of any collected collectors in matchnodes paths, so they
         # are not collected more than once.
@@ -679,31 +680,31 @@ def collect(self) -> Iterator[Union[nodes.Item, nodes.Collector]]:
                         break
 
                     if parent.is_dir():
-                        pkginit = py.path.local(parent / "__init__.py")
-                        if pkginit.isfile() and pkginit not in node_cache1:
+                        pkginit = parent / "__init__.py"
+                        if pkginit.is_file() and pkginit not in node_cache1:
                             col = self._collectfile(pkginit, handle_dupes=False)
                             if col:
                                 if isinstance(col[0], Package):
                                     pkg_roots[str(parent)] = col[0]
-                                node_cache1[col[0].fspath] = [col[0]]
+                                node_cache1[Path(col[0].fspath)] = [col[0]]
 
             # If it's a directory argument, recurse and look for any Subpackages.
             # Let the Package collector deal with subnodes, don't collect here.
             if argpath.is_dir():
                 assert not names, "invalid arg {!r}".format((argpath, names))
 
-                seen_dirs: Set[py.path.local] = set()
+                seen_dirs: Set[Path] = set()
                 for direntry in visit(str(argpath), self._recurse):
                     if not direntry.is_file():
                         continue
 
-                    path = py.path.local(direntry.path)
-                    dirpath = path.dirpath()
+                    path = Path(direntry.path)
+                    dirpath = path.parent
 
                     if dirpath not in seen_dirs:
                         # Collect packages first.
                         seen_dirs.add(dirpath)
-                        pkginit = dirpath.join("__init__.py")
+                        pkginit = dirpath / "__init__.py"
                         if pkginit.exists():
                             for x in self._collectfile(pkginit):
                                 yield x
@@ -714,23 +715,22 @@ def collect(self) -> Iterator[Union[nodes.Item, nodes.Collector]]:
                         continue
 
                     for x in self._collectfile(path):
-                        key = (type(x), x.fspath)
-                        if key in node_cache2:
-                            yield node_cache2[key]
+                        key2 = (type(x), Path(x.fspath))
+                        if key2 in node_cache2:
+                            yield node_cache2[key2]
                         else:
-                            node_cache2[key] = x
+                            node_cache2[key2] = x
                             yield x
             else:
                 assert argpath.is_file()
 
-                argpath_ = py.path.local(argpath)
-                if argpath_ in node_cache1:
-                    col = node_cache1[argpath_]
+                if argpath in node_cache1:
+                    col = node_cache1[argpath]
                 else:
-                    collect_root = pkg_roots.get(argpath_.dirname, self)
-                    col = collect_root._collectfile(argpath_, handle_dupes=False)
+                    collect_root = pkg_roots.get(str(argpath.parent), self)
+                    col = collect_root._collectfile(argpath, handle_dupes=False)
                     if col:
-                        node_cache1[argpath_] = col
+                        node_cache1[argpath] = col
 
                 matching = []
                 work: List[
@@ -846,7 +846,7 @@ def resolve_collection_argument(
 
     This function ensures the path exists, and returns a tuple:
 
-        (py.path.path("/full/path/to/pkg/tests/test_foo.py"), ["TestClass", "test_foo"])
+        (Path("/full/path/to/pkg/tests/test_foo.py"), ["TestClass", "test_foo"])
 
     When as_pypath is True, expects that the command-line argument actually contains
     module paths instead of file-system paths:

src/_pytest/python.py

@@ -66,6 +66,8 @@
 from _pytest.mark.structures import normalize_mark_list
 from _pytest.outcomes import fail
 from _pytest.outcomes import skip
+from _pytest.pathlib import bestrelpath
+from _pytest.pathlib import fnmatch_ex
 from _pytest.pathlib import import_path
 from _pytest.pathlib import ImportPathMismatchError
 from _pytest.pathlib import parts
@@ -190,11 +192,10 @@ def pytest_pyfunc_call(pyfuncitem: "Function") -> Optional[object]:
 def pytest_collect_file(
     fspath: Path, path: py.path.local, parent: nodes.Collector
 ) -> Optional["Module"]:
-    ext = path.ext
-    if ext == ".py":
+    if fspath.suffix == ".py":
         if not parent.session.isinitpath(fspath):
             if not path_matches_patterns(
-                path, parent.config.getini("python_files") + ["__init__.py"]
+                fspath, parent.config.getini("python_files") + ["__init__.py"]
             ):
                 return None
         ihook = parent.session.gethookproxy(fspath)
@@ -205,13 +206,13 @@ def pytest_collect_file(
     return None
 
 
-def path_matches_patterns(path: py.path.local, patterns: Iterable[str]) -> bool:
+def path_matches_patterns(path: Path, patterns: Iterable[str]) -> bool:
     """Return whether path matches any of the patterns in the list of globs given."""
-    return any(path.fnmatch(pattern) for pattern in patterns)
+    return any(fnmatch_ex(pattern, path) for pattern in patterns)
 
 
-def pytest_pycollect_makemodule(path: py.path.local, parent) -> "Module":
-    if path.basename == "__init__.py":
+def pytest_pycollect_makemodule(fspath: Path, path: py.path.local, parent) -> "Module":
+    if fspath.name == "__init__.py":
         pkg: Package = Package.from_parent(parent, fspath=path)
         return pkg
     mod: Module = Module.from_parent(parent, fspath=path)
@@ -677,21 +678,21 @@ def _recurse(self, direntry: "os.DirEntry[str]") -> bool:
         if ihook.pytest_ignore_collect(fspath=fspath, path=path, config=self.config):
             return False
         norecursepatterns = self.config.getini("norecursedirs")
-        if any(path.check(fnmatch=pat) for pat in norecursepatterns):
+        if any(fnmatch_ex(pat, fspath) for pat in norecursepatterns):
             return False
         return True
 
     def _collectfile(
-        self, path: py.path.local, handle_dupes: bool = True
+        self, fspath: Path, handle_dupes: bool = True
     ) -> Sequence[nodes.Collector]:
-        fspath = Path(path)
+        path = py.path.local(fspath)
         assert (
-            path.isfile()
+            fspath.is_file()
         ), "{!r} is not a file (isdir={!r}, exists={!r}, islink={!r})".format(
-            path, path.isdir(), path.exists(), path.islink()
+            path, fspath.is_dir(), fspath.exists(), fspath.is_symlink()
         )
-        ihook = self.session.gethookproxy(path)
-        if not self.session.isinitpath(path):
+        ihook = self.session.gethookproxy(fspath)
+        if not self.session.isinitpath(fspath):
             if ihook.pytest_ignore_collect(
                 fspath=fspath, path=path, config=self.config
             ):
@@ -701,32 +702,32 @@ def _collectfile(
             keepduplicates = self.config.getoption("keepduplicates")
             if not keepduplicates:
                 duplicate_paths = self.config.pluginmanager._duplicatepaths
-                if path in duplicate_paths:
+                if fspath in duplicate_paths:
                     return ()
                 else:
-                    duplicate_paths.add(path)
+                    duplicate_paths.add(fspath)
 
         return ihook.pytest_collect_file(fspath=fspath, path=path, parent=self)  # type: ignore[no-any-return]
 
     def collect(self) -> Iterable[Union[nodes.Item, nodes.Collector]]:
-        this_path = self.fspath.dirpath()
-        init_module = this_path.join("__init__.py")
-        if init_module.check(file=1) and path_matches_patterns(
+        this_path = Path(self.fspath).parent
+        init_module = this_path / "__init__.py"
+        if init_module.is_file() and path_matches_patterns(
             init_module, self.config.getini("python_files")
         ):
-            yield Module.from_parent(self, fspath=init_module)
-        pkg_prefixes: Set[py.path.local] = set()
+            yield Module.from_parent(self, fspath=py.path.local(init_module))
+        pkg_prefixes: Set[Path] = set()
         for direntry in visit(str(this_path), recurse=self._recurse):
-            path = py.path.local(direntry.path)
+            path = Path(direntry.path)
 
             # We will visit our own __init__.py file, in which case we skip it.
             if direntry.is_file():
-                if direntry.name == "__init__.py" and path.dirpath() == this_path:
+                if direntry.name == "__init__.py" and path.parent == this_path:
                     continue
 
             parts_ = parts(direntry.path)
             if any(
-                str(pkg_prefix) in parts_ and pkg_prefix.join("__init__.py") != path
+                str(pkg_prefix) in parts_ and pkg_prefix / "__init__.py" != path
                 for pkg_prefix in pkg_prefixes
             ):
                 continue
@@ -736,7 +737,7 @@ def collect(self) -> Iterable[Union[nodes.Item, nodes.Collector]]:
             elif not direntry.is_dir():
                 # Broken symlink or invalid/missing file.
                 continue
-            elif path.join("__init__.py").check(file=1):
+            elif path.joinpath("__init__.py").is_file():
                 pkg_prefixes.add(path)
 
 
@@ -1416,13 +1417,13 @@ def _show_fixtures_per_test(config: Config, session: Session) -> None:
     import _pytest.config
 
     session.perform_collect()
-    curdir = py.path.local()
+    curdir = Path.cwd()
     tw = _pytest.config.create_terminal_writer(config)
     verbose = config.getvalue("verbose")
 
-    def get_best_relpath(func):
+    def get_best_relpath(func) -> str:
         loc = getlocation(func, str(curdir))
-        return curdir.bestrelpath(py.path.local(loc))
+        return bestrelpath(curdir, Path(loc))
 
     def write_fixture(fixture_def: fixtures.FixtureDef[object]) -> None:
         argname = fixture_def.argname
@@ -1472,7 +1473,7 @@ def _showfixtures_main(config: Config, session: Session) -> None:
     import _pytest.config
 
     session.perform_collect()
-    curdir = py.path.local()
+    curdir = Path.cwd()
     tw = _pytest.config.create_terminal_writer(config)
     verbose = config.getvalue("verbose")
 
@@ -1494,7 +1495,7 @@ def _showfixtures_main(config: Config, session: Session) -> None:
                 (
                     len(fixturedef.baseid),
                     fixturedef.func.__module__,
-                    curdir.bestrelpath(py.path.local(loc)),
+                    bestrelpath(curdir, Path(loc)),
                     fixturedef.argname,
                     fixturedef,
                 )

testing/test_collection.py

@@ -212,12 +212,12 @@ def test__in_venv(self, pytester: Pytester, fname: str) -> None:
         bindir = "Scripts" if sys.platform.startswith("win") else "bin"
         # no bin/activate, not a virtualenv
         base_path = pytester.mkdir("venv")
-        assert _in_venv(py.path.local(base_path)) is False
+        assert _in_venv(base_path) is False
         # with bin/activate, totally a virtualenv
         bin_path = base_path.joinpath(bindir)
         bin_path.mkdir()
         bin_path.joinpath(fname).touch()
-        assert _in_venv(py.path.local(base_path)) is True
+        assert _in_venv(base_path) is True
 
     def test_custom_norecursedirs(self, pytester: Pytester) -> None:
         pytester.makeini(