Merge pull request #10416 from sbidoul/test-indented-warnings-sbi

Detect indented ERROR and WARNING messages in tests and ignore own deprecation warnings

Author: sbidoul

tests/functional/test_download.py

@@ -1116,9 +1116,17 @@ def test_download_file_url_existing_bad_download(
     simple_pkg_bytes = simple_pkg.read_bytes()
     url = f"{simple_pkg.as_uri()}#sha256={sha256(simple_pkg_bytes).hexdigest()}"
 
-    shared_script.pip("download", "-d", str(download_dir), url)
+    result = shared_script.pip(
+        "download",
+        "-d",
+        str(download_dir),
+        url,
+        allow_stderr_warning=True,  # bad hash
+    )
 
     assert simple_pkg_bytes == downloaded_path.read_bytes()
+    assert "WARNING: Previously-downloaded file" in result.stderr
+    assert "has bad hash. Re-downloading." in result.stderr
 
 
 def test_download_http_url_bad_hash(
@@ -1144,9 +1152,17 @@ def test_download_http_url_bad_hash(
     base_address = f"http://{mock_server.host}:{mock_server.port}"
     url = f"{base_address}/simple-1.0.tar.gz#sha256={digest}"
 
-    shared_script.pip("download", "-d", str(download_dir), url)
+    result = shared_script.pip(
+        "download",
+        "-d",
+        str(download_dir),
+        url,
+        allow_stderr_warning=True,  # bad hash
+    )
 
     assert simple_pkg_bytes == downloaded_path.read_bytes()
+    assert "WARNING: Previously-downloaded file" in result.stderr
+    assert "has bad hash. Re-downloading." in result.stderr
 
     mock_server.stop()
     requests = mock_server.get_requests()

tests/functional/test_install.py

@@ -183,7 +183,16 @@ def test_pep518_with_user_pip(
     non-isolated environment, and break pip in the system site-packages,
     so that isolated uses of pip will fail.
     """
-    script.pip("install", "--ignore-installed", "-f", common_wheels, "--user", pip_src)
+    script.pip(
+        "install",
+        "--ignore-installed",
+        "-f",
+        common_wheels,
+        "--user",
+        pip_src,
+        # WARNING: The scripts pip, pip3, ... are installed in ... which is not on PATH
+        allow_stderr_warning=True,
+    )
     system_pip_dir = script.site_packages_path / "pip"
     assert not system_pip_dir.exists()
     system_pip_dir.mkdir()
@@ -1542,13 +1551,16 @@ def test_install_topological_sort(script: PipTestEnvironment, data: TestData) ->
 
 @pytest.mark.usefixtures("with_wheel")
 def test_install_wheel_broken(script: PipTestEnvironment) -> None:
-    res = script.pip_install_local("wheelbroken", expect_stderr=True)
+    res = script.pip_install_local("wheelbroken", allow_stderr_error=True)
+    assert "ERROR: Failed building wheel for wheelbroken" in res.stderr
+    # Fallback to setup.py install (https://github.com/pypa/pip/issues/8368)
     assert "Successfully installed wheelbroken-0.1" in str(res), str(res)
 
 
 @pytest.mark.usefixtures("with_wheel")
 def test_cleanup_after_failed_wheel(script: PipTestEnvironment) -> None:
-    res = script.pip_install_local("wheelbrokenafter", expect_stderr=True)
+    res = script.pip_install_local("wheelbrokenafter", allow_stderr_error=True)
+    assert "ERROR: Failed building wheel for wheelbrokenafter" in res.stderr
     # One of the effects of not cleaning up is broken scripts:
     script_py = script.bin_path / "script.py"
     assert script_py.exists(), script_py
@@ -1577,7 +1589,12 @@ def test_install_builds_wheels(script: PipTestEnvironment, data: TestData) -> No
     # vcs coverage.
     to_install = data.packages.joinpath("requires_wheelbroken_upper")
     res = script.pip(
-        "install", "--no-index", "-f", data.find_links, to_install, expect_stderr=True
+        "install",
+        "--no-index",
+        "-f",
+        data.find_links,
+        to_install,
+        allow_stderr_error=True,  # error building wheelbroken
     )
     expected = (
         "Successfully installed requires-wheelbroken-upper-0"
@@ -1620,7 +1637,7 @@ def test_install_no_binary_disables_building_wheels(
         "-f",
         data.find_links,
         to_install,
-        expect_stderr=True,
+        allow_stderr_error=True,  # error building wheelbroken
     )
     expected = (
         "Successfully installed requires-wheelbroken-upper-0"

tests/functional/test_install_vcs_git.py

@@ -94,7 +94,7 @@ def _install_version_pkg_only(
     script: PipTestEnvironment,
     path: Path,
     rev: Optional[str] = None,
-    expect_stderr: bool = False,
+    allow_stderr_warning: bool = False,
 ) -> None:
     """
     Install the version_pkg package in editable mode (without returning
@@ -106,14 +106,16 @@ def _install_version_pkg_only(
       rev: an optional revision to install like a branch name or tag.
     """
     version_pkg_url = _make_version_pkg_url(path, rev=rev)
-    script.pip("install", "-e", version_pkg_url, expect_stderr=expect_stderr)
+    script.pip(
+        "install", "-e", version_pkg_url, allow_stderr_warning=allow_stderr_warning
+    )
 
 
 def _install_version_pkg(
     script: PipTestEnvironment,
     path: Path,
     rev: Optional[str] = None,
-    expect_stderr: bool = False,
+    allow_stderr_warning: bool = False,
 ) -> str:
     """
     Install the version_pkg package in editable mode, and return the version
@@ -128,7 +130,7 @@ def _install_version_pkg(
         script,
         path,
         rev=rev,
-        expect_stderr=expect_stderr,
+        allow_stderr_warning=allow_stderr_warning,
     )
     result = script.run("version_pkg")
     version = result.stdout.strip()
@@ -227,7 +229,13 @@ def test_git_with_short_sha1_revisions(script: PipTestEnvironment) -> None:
         "HEAD~1",
         cwd=version_pkg_path,
     ).stdout.strip()[:7]
-    version = _install_version_pkg(script, version_pkg_path, rev=sha1)
+    version = _install_version_pkg(
+        script,
+        version_pkg_path,
+        rev=sha1,
+        # WARNING: Did not find branch or tag ..., assuming revision or ref.
+        allow_stderr_warning=True,
+    )
     assert "0.1" == version
 
 
@@ -273,6 +281,8 @@ def test_git_install_ref(script: PipTestEnvironment) -> None:
         script,
         version_pkg_path,
         rev="refs/foo/bar",
+        # WARNING: Did not find branch or tag ..., assuming revision or ref.
+        allow_stderr_warning=True,
     )
     assert "0.1" == version
 
@@ -294,6 +304,8 @@ def test_git_install_then_install_ref(script: PipTestEnvironment) -> None:
         script,
         version_pkg_path,
         rev="refs/foo/bar",
+        # WARNING: Did not find branch or tag ..., assuming revision or ref.
+        allow_stderr_warning=True,
     )
     assert "0.1" == version
 

tests/functional/test_uninstall.py

@@ -684,7 +684,7 @@ def test_uninstall_editable_and_pip_install_easy_install_remove(
     os.remove(pip_test_fspkg_pth)
 
     # Uninstall will fail with given warning
-    uninstall = script.pip("uninstall", "FSPkg", "-y")
+    uninstall = script.pip("uninstall", "FSPkg", "-y", allow_stderr_warning=True)
     assert "Cannot remove entries from nonexistent file" in uninstall.stderr
 
     assert (

tests/lib/__init__.py

@@ -39,7 +39,6 @@
 from pip._internal.models.selection_prefs import SelectionPreferences
 from pip._internal.models.target_python import TargetPython
 from pip._internal.network.session import PipSession
-from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX
 from tests.lib.venv import VirtualEnvironment
 from tests.lib.wheel import make_wheel
 
@@ -447,6 +446,7 @@ def _check_stderr(
 
     lines = stderr.splitlines()
     for line in lines:
+        line = line.lstrip()
         # First check for logging errors, which we don't allow during
         # tests even if allow_stderr_error=True (since a logging error
         # would signal a bug in pip's code).
@@ -473,7 +473,7 @@ def _check_stderr(
         if allow_stderr_warning:
             continue
 
-        if line.startswith("WARNING: ") or line.startswith(DEPRECATION_MSG_PREFIX):
+        if line.startswith("WARNING: "):
             reason = (
                 "stderr has an unexpected warning "
                 "(pass allow_stderr_warning=True to permit this)"

tests/lib/test_lib.py

@@ -150,7 +150,6 @@ def test_run__allow_stderr_warning(self, script: PipTestEnvironment) -> None:
     @pytest.mark.parametrize(
         "prefix",
         (
-            "DEPRECATION",
             "WARNING",
             "ERROR",
         ),
@@ -167,7 +166,6 @@ def test_run__allow_stderr_error(
     @pytest.mark.parametrize(
         "prefix, expected_start",
         (
-            ("DEPRECATION", "stderr has an unexpected warning"),
             ("WARNING", "stderr has an unexpected warning"),
             ("ERROR", "stderr has an unexpected error"),
         ),