linkcheck builder: optionally allow HTTP 401 status code hyperlinks to be reported broken

CHANGES.rst

@@ -18,6 +18,9 @@ Bugs fixed
 
 * #11668: Raise a useful error when ``theme.conf`` is missing.
   Patch by Vinay Sajip.
+* #11433: The ``linkcheck_allow_unauthorized`` configuration option (default
+  value ``true``) has been added.  Set this option to ``false`` to handle
+  HTTP 401 (unauthorized) server responses as broken hyperlinks.
 
 Testing
 -------
@@ -304,7 +307,6 @@ Features added
 
 Bugs fixed
 ----------
-
 * Restored the ``footnote-reference`` class that has been removed in
   the latest (unreleased) version of Docutils.
 * #11486: Use :rfc:`8081` font file MIME types in the EPUB builder.

doc/usage/configuration.rst

@@ -2915,6 +2915,14 @@ Options for the linkcheck builder
 
    .. versionadded:: 4.4
 
+.. confval:: linkcheck_allow_unauthorized
+
+   When a webserver responds with an HTTP 401 (unauthorized) response, the
+   default behaviour is to treat the link as "working".  To change that
+   behaviour, set this option to ``False``.
+
+   .. versionadded:: 7.3
+
 
 Options for the XML builder
 ---------------------------

sphinx/builders/linkcheck.py

@@ -283,6 +283,7 @@ def __init__(self, config: Config,
         self.allowed_redirects = config.linkcheck_allowed_redirects
         self.retries: int = config.linkcheck_retries
         self.rate_limit_timeout = config.linkcheck_rate_limit_timeout
+        self.allow_unauthorized = config.linkcheck_allow_unauthorized
 
         self.user_agent = config.user_agent
         self.tls_verify = config.tls_verify
@@ -437,9 +438,10 @@ def _check_uri(self, uri: str, hyperlink: Hyperlink) -> tuple[str, str, int]:
             except HTTPError as err:
                 error_message = str(err)
 
-                # Unauthorised: the reference probably exists
+                # Unauthorized: the client did not provide required credentials
                 if status_code == 401:
-                    return 'working', 'unauthorized', 0
+                    status = 'working' if self.allow_unauthorized else 'broken'
+                    return status, 'unauthorized', 0
 
                 # Rate limiting; back-off if allowed, or report failure otherwise
                 if status_code == 429:
@@ -625,6 +627,7 @@ def setup(app: Sphinx) -> dict[str, Any]:
     app.add_config_value('linkcheck_anchors_ignore', ['^!'], False)
     app.add_config_value('linkcheck_anchors_ignore_for_url', (), False, (tuple, list))
     app.add_config_value('linkcheck_rate_limit_timeout', 300.0, False)
+    app.add_config_value('linkcheck_allow_unauthorized', True, False)
 
     app.add_event('linkcheck-process-uri')
 

tests/test_build_linkcheck.py

@@ -343,8 +343,12 @@ class CustomHandler(http.server.BaseHTTPRequestHandler):
 
         def authenticated(method):
             def method_if_authenticated(self):
-                if (expected_token is None
-                        or self.headers["Authorization"] == f"Basic {expected_token}"):
+                if expected_token is None:
+                    return method(self)
+                elif not self.headers["Authorization"]:
+                    self.send_response(401, "Unauthorized")
+                    self.end_headers()
+                elif self.headers["Authorization"] == f"Basic {expected_token}":
                     return method(self)
                 else:
                     self.send_response(403, "Forbidden")
@@ -387,6 +391,20 @@ def test_auth_header_uses_first_match(app):
     assert content["status"] == "working"
 
 
+@pytest.mark.sphinx(
+    'linkcheck', testroot='linkcheck-localserver', freshenv=True,
+    confoverrides={'linkcheck_allow_unauthorized': False})
+def test_unauthorized_broken(app):
+    with http_server(custom_handler(valid_credentials=("user1", "password"))):
+        app.build()
+
+    with open(app.outdir / "output.json", encoding="utf-8") as fp:
+        content = json.load(fp)
+
+    assert content["info"] == "unauthorized"
+    assert content["status"] == "broken"
+
+
 @pytest.mark.sphinx(
     'linkcheck', testroot='linkcheck-localserver', freshenv=True,
     confoverrides={'linkcheck_auth': [(r'^$', ('user1', 'password'))]})
@@ -397,10 +415,9 @@ def test_auth_header_no_match(app):
     with open(app.outdir / "output.json", encoding="utf-8") as fp:
         content = json.load(fp)
 
-    # TODO: should this test's webserver return HTTP 401 here?
-    # https://github.com/sphinx-doc/sphinx/issues/11433
-    assert content["info"] == "403 Client Error: Forbidden for url: http://localhost:7777/"
-    assert content["status"] == "broken"
+    # This link is considered working based on the default linkcheck_allow_unauthorized=true
+    assert content["info"] == "unauthorized"
+    assert content["status"] == "working"
 
 
 @pytest.mark.sphinx(