linkcheck: Use context managers for HTTP requests (#11318)

This closes HTTP responses when no content reads are required, as
when requests are made in streaming mode, ``requests`` doesn't know
whether the caller may intend to later read content from a streamed
HTTP response object and holds the socket open.

Co-authored-by: Adam Turner <[email protected]>

Author: jayaddison

.github/workflows/main.yml

@@ -13,7 +13,7 @@ env:
   FORCE_COLOR: "1"
   PYTHONDEVMODE: "1"  # -X dev
   PYTHONWARNDEFAULTENCODING: "1"  # -X warn_default_encoding
-  PYTHONWARNINGS: "error,always:unclosed:ResourceWarning::"  # default: all warnings as errors, except ResourceWarnings about unclosed items
+  PYTHONWARNINGS: "error"  # default: all warnings as errors
 
 jobs:
   ubuntu:

sphinx/builders/linkcheck.py

@@ -316,32 +316,30 @@ def check_uri() -> tuple[str, str, int]:
             try:
                 if anchor and self.config.linkcheck_anchors:
                     # Read the whole document and see if #anchor exists
-                    response = requests.get(req_url, stream=True, config=self.config,
-                                            auth=auth_info, **kwargs)
-                    response.raise_for_status()
-                    found = check_anchor(response, unquote(anchor))
+                    with requests.get(req_url, stream=True, config=self.config, auth=auth_info,
+                                      **kwargs) as response:
+                        response.raise_for_status()
+                        found = check_anchor(response, unquote(anchor))
 
                     if not found:
                         raise Exception(__("Anchor '%s' not found") % anchor)
                 else:
                     try:
                         # try a HEAD request first, which should be easier on
                         # the server and the network
-                        response = requests.head(req_url, allow_redirects=True,
-                                                 config=self.config, auth=auth_info,
-                                                 **kwargs)
-                        response.raise_for_status()
+                        with requests.head(req_url, allow_redirects=True, config=self.config,
+                                           auth=auth_info, **kwargs) as response:
+                            response.raise_for_status()
                     # Servers drop the connection on HEAD requests, causing
                     # ConnectionError.
                     except (ConnectionError, HTTPError, TooManyRedirects) as err:
                         if isinstance(err, HTTPError) and err.response.status_code == 429:
                             raise
                         # retry with GET request if that fails, some servers
                         # don't like HEAD requests.
-                        response = requests.get(req_url, stream=True,
-                                                config=self.config,
-                                                auth=auth_info, **kwargs)
-                        response.raise_for_status()
+                        with requests.get(req_url, stream=True, config=self.config,
+                                          auth=auth_info, **kwargs) as response:
+                            response.raise_for_status()
             except HTTPError as err:
                 if err.response.status_code == 401:
                     # We'll take "Unauthorized" as working.

tests/roots/test-linkcheck-anchors-ignore/conf.py

@@ -1,3 +1,3 @@
 exclude_patterns = ['_build']
 linkcheck_anchors = True
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05

tests/roots/test-linkcheck-documents_exclude/conf.py

@@ -3,4 +3,4 @@
     '^broken_link$',
     'br[0-9]ken_link',
 ]
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05

tests/roots/test-linkcheck-localserver-anchor/conf.py

@@ -1,3 +1,3 @@
 exclude_patterns = ['_build']
 linkcheck_anchors = True
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05

tests/roots/test-linkcheck-localserver-https/conf.py

@@ -1,2 +1,2 @@
 exclude_patterns = ['_build']
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05

tests/roots/test-linkcheck-localserver-warn-redirects/conf.py

@@ -1,2 +1,2 @@
 exclude_patterns = ['_build']
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05

tests/roots/test-linkcheck-localserver/conf.py

@@ -1,2 +1,2 @@
 exclude_patterns = ['_build']
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05

tests/roots/test-linkcheck-raw-node/conf.py

@@ -1,2 +1,2 @@
 exclude_patterns = ['_build']
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05

tests/roots/test-linkcheck-too-many-retries/conf.py

@@ -1,3 +1,3 @@
 exclude_patterns = ['_build']
 linkcheck_anchors = True
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05