test: Enforce network hygiene with pytest-subket (#13481)

pytest-subket is my fork of pytest-socket with changes that allow it to
function in subprocesses and even across environments (as long as the
pytest-subket package is installed, no dependencies needed).

To promote better test hygiene, tests should need to opt-in to access
the Internet. This is managed via the network marker, but is not
enforced. With pytest-subket, tests accessing the Internet w/o the
marker will immediately fail.

This will encourage us to write tests to avoid hitting the network which
will result in a faster, more reliable test suite that won't regress.
This also makes the lives of downstream that run our test suite in a
sandboxed environment nicer as our network marker should be up to date
24/7 now (barring any tests using remote VCS repositories).

* test: Bring network marker up to date with reality

All instances of the network marker were removed. Afterwards a full test
suite was performed with pytest-subket active. The marker was reapplied
to the tests that actually needed it (including tests that hit the
network via Git or other VCS software).

Author: ichard26

pyproject.toml

@@ -53,10 +53,12 @@ test = [
     "cryptography",
     "freezegun",
     "installer",
-    "pytest",
+    # pytest-subket requires 7.0+
+    "pytest >= 7.0",
     "pytest-cov",
     "pytest-rerunfailures",
     "pytest-xdist",
+    "pytest-subket",
     "scripttest",
     "setuptools",
     # macOS (darwin) arm64 always uses virtualenv >= 20.0
@@ -76,6 +78,7 @@ test-common-wheels = [
     "wheel",
     # As required by pytest-cov.
     "coverage >= 4.4",
+    "pytest-subket >= 0.8.1",
 ]
 
 [tool.setuptools]
@@ -296,7 +299,7 @@ follow_imports = "skip"
 #
 
 [tool.pytest.ini_options]
-addopts = "--ignore src/pip/_vendor --ignore tests/tests_cache -r aR --color=yes"
+addopts = "--ignore src/pip/_vendor --ignore tests/tests_cache -r aR --color=yes --disable-socket --allow-hosts=localhost"
 xfail_strict = true
 markers = [
     "network: tests that need network",

tests/conftest.py

@@ -102,6 +102,10 @@ def pytest_collection_modifyitems(config: Config, items: list[pytest.Function])
         if item.get_closest_marker("search") and not config.getoption("--run-search"):
             item.add_marker(pytest.mark.skip("pip search test skipped"))
 
+        # Exempt tests known to use the network from pytest-subket.
+        if item.get_closest_marker("network") is not None:
+            item.add_marker(pytest.mark.enable_socket)
+
         if "CI" in os.environ:
             # Mark network tests as flaky
             if item.get_closest_marker("network") is not None:
@@ -447,6 +451,18 @@ def coverage_install(
     return _common_wheel_editable_install(tmpdir_factory, common_wheels, "coverage")
 
 
+@pytest.fixture(scope="session")
+def socket_install(tmpdir_factory: pytest.TempPathFactory, common_wheels: Path) -> Path:
+    lib_dir = _common_wheel_editable_install(
+        tmpdir_factory, common_wheels, "pytest_subket"
+    )
+    # pytest-subket is only included so it can intercept and block unexpected
+    # network requests. It should NOT be visible to the pip under test.
+    dist_info = next(lib_dir.glob("*.dist-info"))
+    shutil.rmtree(dist_info)
+    return lib_dir
+
+
 def install_pth_link(
     venv: VirtualEnvironment, project_name: str, lib_dir: Path
 ) -> None:
@@ -464,6 +480,7 @@ def virtualenv_template(
     setuptools_install: Path,
     wheel_install: Path,
     coverage_install: Path,
+    socket_install: Path,
 ) -> VirtualEnvironment:
     venv_type: VirtualEnvironmentType
     if request.config.getoption("--use-venv"):
@@ -475,9 +492,13 @@ def virtualenv_template(
     tmpdir = tmpdir_factory.mktemp("virtualenv")
     venv = VirtualEnvironment(tmpdir.joinpath("venv_orig"), venv_type=venv_type)
 
-    # Install setuptools, wheel and pip.
+    # Install setuptools, wheel, pytest-subket, and pip.
     install_pth_link(venv, "setuptools", setuptools_install)
     install_pth_link(venv, "wheel", wheel_install)
+    install_pth_link(venv, "pytest_subket", socket_install)
+    # Also copy pytest-subket's .pth file so it can intercept socket calls.
+    with open(venv.site / "pytest_socket.pth", "w") as f:
+        f.write(socket_install.joinpath("pytest_socket.pth").read_text())
 
     pth, dist_info = pip_editable_parts
 

tests/functional/test_download.py

@@ -1101,7 +1101,9 @@ def test_download_file_url_existing_ok_download(
     simple_pkg = shared_data.packages / "simple-1.0.tar.gz"
     url = f"{simple_pkg.as_uri()}#sha256={sha256(downloaded_path_bytes).hexdigest()}"
 
-    shared_script.pip("download", "-d", str(download_dir), url)
+    shared_script.pip(
+        "download", "-d", str(download_dir), url, "--disable-pip-version-check"
+    )
 
     assert downloaded_path_bytes == downloaded_path.read_bytes()
 

tests/functional/test_install.py

@@ -155,6 +155,7 @@ def test_pep518_refuses_invalid_build_system(
     assert "pyproject.toml" in result.stderr
 
 
+@pytest.mark.network
 def test_pep518_allows_missing_requires(
     script: PipTestEnvironment, data: TestData, common_wheels: Path
 ) -> None:
@@ -175,6 +176,7 @@ def test_pep518_allows_missing_requires(
     assert result.files_created
 
 
+@pytest.mark.network
 @pytest.mark.usefixtures("enable_user_site")
 def test_pep518_with_user_pip(
     script: PipTestEnvironment, pip_src: Path, data: TestData, common_wheels: Path
@@ -2424,6 +2426,7 @@ def test_install_sends_certs_for_pep518_deps(
         assert environ.get("SSL_CLIENT_CERT", "")
 
 
+@pytest.mark.network
 def test_install_skip_work_dir_pkg(script: PipTestEnvironment, data: TestData) -> None:
     """
     Test that install of a package in working directory

tests/functional/test_install_compat.py

@@ -6,8 +6,6 @@
 import os
 from pathlib import Path
 
-import pytest
-
 from tests.lib import (
     PipTestEnvironment,
     TestData,
@@ -16,7 +14,6 @@
 )
 
 
-@pytest.mark.network
 def test_debian_egg_name_workaround(
     script: PipTestEnvironment,
     shared_data: TestData,

tests/functional/test_install_extras.py

@@ -188,10 +188,11 @@ def test_install_special_extra(
     ) in result.stderr, str(result)
 
 
+@pytest.mark.network
 def test_install_requirements_no_r_flag(script: PipTestEnvironment) -> None:
     """Beginners sometimes forget the -r and this leads to confusion"""
     result = script.pip("install", "requirements.txt", expect_error=True)
-    assert 'literally named "requirements.txt"' in result.stdout
+    assert 'literally named "requirements.txt"' in result.stdout, str(result)
 
 
 @pytest.mark.parametrize(

tests/functional/test_install_report.py

@@ -117,6 +117,7 @@ def test_skipped_yanked_version(
     assert simple_report["metadata"]["version"] == "2.0"
 
 
+@pytest.mark.network
 @pytest.mark.parametrize(
     "specifiers",
     [
@@ -127,7 +128,6 @@ def test_skipped_yanked_version(
         ("Paste[openid]==1.7.5.1", "Paste==1.7.5.1"),
     ],
 )
-@pytest.mark.network
 def test_install_report_index(
     script: PipTestEnvironment, tmp_path: Path, specifiers: tuple[str, ...]
 ) -> None:
@@ -178,7 +178,6 @@ def test_install_report_index_multiple_extras(
     assert install_dict["paste"]["requested_extras"] == ["openid", "subprocess"]
 
 
-@pytest.mark.network
 def test_install_report_direct_archive(
     script: PipTestEnvironment, tmp_path: Path, shared_data: TestData
 ) -> None:
@@ -297,7 +296,6 @@ def test_install_report_vcs_editable(
     assert pip_test_package_report["download_info"]["dir_info"]["editable"] is True
 
 
-@pytest.mark.network
 def test_install_report_local_path_with_extras(
     script: PipTestEnvironment, tmp_path: Path, shared_data: TestData
 ) -> None:
@@ -342,7 +340,6 @@ def test_install_report_local_path_with_extras(
     assert "requested_extras" not in simple_report
 
 
-@pytest.mark.network
 def test_install_report_editable_local_path_with_extras(
     script: PipTestEnvironment, tmp_path: Path, shared_data: TestData
 ) -> None:

tests/unit/test_finder.py

@@ -81,7 +81,6 @@ def test_incorrect_case_file_index(data: TestData) -> None:
     assert found.link.url.endswith("Dinner-2.0.tar.gz")
 
 
-@pytest.mark.network
 def test_finder_detects_latest_already_satisfied_find_links(data: TestData) -> None:
     """Test PackageFinder detects latest already satisfied using find-links"""
     req = install_req_from_line("simple")
@@ -98,7 +97,6 @@ def test_finder_detects_latest_already_satisfied_find_links(data: TestData) -> N
         finder.find_requirement(req, True)
 
 
-@pytest.mark.network
 def test_finder_detects_latest_already_satisfied_pypi_links() -> None:
     """Test PackageFinder detects latest already satisfied using pypi links"""
     req = install_req_from_line("initools")