linkcheck: Use context managers for HTTP requests (#11318)

This closes HTTP responses when no content reads are required, as
when requests are made in streaming mode, ``requests`` doesn't know
whether the caller may intend to later read content from a streamed
HTTP response object and holds the socket open.

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>

13 files changed, 38 insertions, 24 deletions

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d96481d6b..2efc1c184 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -13,7 +13,7 @@ env:
   FORCE_COLOR: "1"
   PYTHONDEVMODE: "1"  # -X dev
   PYTHONWARNDEFAULTENCODING: "1"  # -X warn_default_encoding
-  PYTHONWARNINGS: "error,always:unclosed:ResourceWarning::"  # default: all warnings as errors, except ResourceWarnings about unclosed items
+  PYTHONWARNINGS: "error"  # default: all warnings as errors
 
 jobs:
   ubuntu:
diff --git a/sphinx/builders/linkcheck.py b/sphinx/builders/linkcheck.py
index e828fc209..6c99f96e6 100644
--- a/sphinx/builders/linkcheck.py
+++ b/sphinx/builders/linkcheck.py
@@ -316,10 +316,10 @@ class HyperlinkAvailabilityCheckWorker(Thread):
             try:
                 if anchor and self.config.linkcheck_anchors:
                     # Read the whole document and see if #anchor exists
-                    response = requests.get(req_url, stream=True, config=self.config,
-                                            auth=auth_info, **kwargs)
-                    response.raise_for_status()
-                    found = check_anchor(response, unquote(anchor))
+                    with requests.get(req_url, stream=True, config=self.config, auth=auth_info,
+                                      **kwargs) as response:
+                        response.raise_for_status()
+                        found = check_anchor(response, unquote(anchor))
 
                     if not found:
                         raise Exception(__("Anchor '%s' not found") % anchor)
@@ -327,10 +327,9 @@ class HyperlinkAvailabilityCheckWorker(Thread):
                     try:
                         # try a HEAD request first, which should be easier on
                         # the server and the network
-                        response = requests.head(req_url, allow_redirects=True,
-                                                 config=self.config, auth=auth_info,
-                                                 **kwargs)
-                        response.raise_for_status()
+                        with requests.head(req_url, allow_redirects=True, config=self.config,
+                                           auth=auth_info, **kwargs) as response:
+                            response.raise_for_status()
                     # Servers drop the connection on HEAD requests, causing
                     # ConnectionError.
                     except (ConnectionError, HTTPError, TooManyRedirects) as err:
@@ -338,10 +337,9 @@ class HyperlinkAvailabilityCheckWorker(Thread):
                             raise
                         # retry with GET request if that fails, some servers
                         # don't like HEAD requests.
-                        response = requests.get(req_url, stream=True,
-                                                config=self.config,
-                                                auth=auth_info, **kwargs)
-                        response.raise_for_status()
+                        with requests.get(req_url, stream=True, config=self.config,
+                                          auth=auth_info, **kwargs) as response:
+                            response.raise_for_status()
             except HTTPError as err:
                 if err.response.status_code == 401:
                     # We'll take "Unauthorized" as working.
diff --git a/tests/roots/test-linkcheck-anchors-ignore/conf.py b/tests/roots/test-linkcheck-anchors-ignore/conf.py
index e3a1918fe..0005bfada 100644
--- a/tests/roots/test-linkcheck-anchors-ignore/conf.py
+++ b/tests/roots/test-linkcheck-anchors-ignore/conf.py
@@ -1,3 +1,3 @@
 exclude_patterns = ['_build']
 linkcheck_anchors = True
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck-documents_exclude/conf.py b/tests/roots/test-linkcheck-documents_exclude/conf.py
index b6dfb26a4..52388f93a 100644
--- a/tests/roots/test-linkcheck-documents_exclude/conf.py
+++ b/tests/roots/test-linkcheck-documents_exclude/conf.py
@@ -3,4 +3,4 @@ linkcheck_exclude_documents = [
     '^broken_link$',
     'br[0-9]ken_link',
 ]
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck-localserver-anchor/conf.py b/tests/roots/test-linkcheck-localserver-anchor/conf.py
index e3a1918fe..0005bfada 100644
--- a/tests/roots/test-linkcheck-localserver-anchor/conf.py
+++ b/tests/roots/test-linkcheck-localserver-anchor/conf.py
@@ -1,3 +1,3 @@
 exclude_patterns = ['_build']
 linkcheck_anchors = True
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck-localserver-https/conf.py b/tests/roots/test-linkcheck-localserver-https/conf.py
index 25747bbc0..a2ce01e65 100644
--- a/tests/roots/test-linkcheck-localserver-https/conf.py
+++ b/tests/roots/test-linkcheck-localserver-https/conf.py
@@ -1,2 +1,2 @@
 exclude_patterns = ['_build']
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck-localserver-warn-redirects/conf.py b/tests/roots/test-linkcheck-localserver-warn-redirects/conf.py
index 25747bbc0..a2ce01e65 100644
--- a/tests/roots/test-linkcheck-localserver-warn-redirects/conf.py
+++ b/tests/roots/test-linkcheck-localserver-warn-redirects/conf.py
@@ -1,2 +1,2 @@
 exclude_patterns = ['_build']
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck-localserver/conf.py b/tests/roots/test-linkcheck-localserver/conf.py
index 25747bbc0..a2ce01e65 100644
--- a/tests/roots/test-linkcheck-localserver/conf.py
+++ b/tests/roots/test-linkcheck-localserver/conf.py
@@ -1,2 +1,2 @@
 exclude_patterns = ['_build']
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck-raw-node/conf.py b/tests/roots/test-linkcheck-raw-node/conf.py
index 25747bbc0..a2ce01e65 100644
--- a/tests/roots/test-linkcheck-raw-node/conf.py
+++ b/tests/roots/test-linkcheck-raw-node/conf.py
@@ -1,2 +1,2 @@
 exclude_patterns = ['_build']
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck-too-many-retries/conf.py b/tests/roots/test-linkcheck-too-many-retries/conf.py
index e3a1918fe..0005bfada 100644
--- a/tests/roots/test-linkcheck-too-many-retries/conf.py
+++ b/tests/roots/test-linkcheck-too-many-retries/conf.py
@@ -1,3 +1,3 @@
 exclude_patterns = ['_build']
 linkcheck_anchors = True
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck/conf.py b/tests/roots/test-linkcheck/conf.py
index 0c197b6f0..6ddb41a5d 100644
--- a/tests/roots/test-linkcheck/conf.py
+++ b/tests/roots/test-linkcheck/conf.py
@@ -1,4 +1,4 @@
 root_doc = 'links'
 exclude_patterns = ['_build']
 linkcheck_anchors = True
-linkcheck_timeout = 0.075
+linkcheck_timeout = 0.05
diff --git a/tests/roots/test-linkcheck/links.rst b/tests/roots/test-linkcheck/links.rst
index 49afba2b3..88c757e03 100644
--- a/tests/roots/test-linkcheck/links.rst
+++ b/tests/roots/test-linkcheck/links.rst
@@ -11,3 +11,4 @@ Some additional anchors to exercise ignore code
 .. image:: http://localhost:7777/image.png
 .. figure:: http://localhost:7777/image2.png
 
+* `Valid anchored url <http://localhost:7777/anchor.html#found>`_
diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py
index acfebd649..260cf2c42 100644
--- a/tests/test_build_linkcheck.py
+++ b/tests/test_build_linkcheck.py
@@ -31,6 +31,9 @@ class DefaultsHandler(http.server.BaseHTTPRequestHandler):
         if self.path[1:].rstrip() == "":
             self.send_response(200, "OK")
             self.end_headers()
+        elif self.path[1:].rstrip() == "anchor.html":
+            self.send_response(200, "OK")
+            self.end_headers()
         else:
             self.send_response(404, "Not Found")
             self.end_headers()
@@ -39,6 +42,9 @@ class DefaultsHandler(http.server.BaseHTTPRequestHandler):
         self.do_HEAD()
         if self.path[1:].rstrip() == "":
             self.wfile.write(b"ok\n\n")
+        elif self.path[1:].rstrip() == "anchor.html":
+            doc = '<!DOCTYPE html><html><body><a id="found"></a></body></html>'
+            self.wfile.write(doc.encode('utf-8'))
 
 
 @pytest.mark.sphinx('linkcheck', testroot='linkcheck', freshenv=True)
@@ -69,8 +75,8 @@ def test_defaults(app):
     for attr in ("filename", "lineno", "status", "code", "uri", "info"):
         assert attr in row
 
-    assert len(content.splitlines()) == 9
-    assert len(rows) == 9
+    assert len(content.splitlines()) == 10
+    assert len(rows) == 10
     # the output order of the rows is not stable
     # due to possible variance in network latency
     rowsby = {row["uri"]: row for row in rows}
@@ -95,6 +101,15 @@ def test_defaults(app):
     assert rowsby["http://localhost:7777#does-not-exist"]["info"] == "Anchor 'does-not-exist' not found"
     # images should fail
     assert "Not Found for url: http://localhost:7777/image.png" in rowsby["http://localhost:7777/image.png"]["info"]
+    # anchor should be found
+    assert rowsby['http://localhost:7777/anchor.html#found'] == {
+        'filename': 'links.rst',
+        'lineno': 14,
+        'status': 'working',
+        'code': 0,
+        'uri': 'http://localhost:7777/anchor.html#found',
+        'info': '',
+    }
 
 
 @pytest.mark.sphinx('linkcheck', testroot='linkcheck-too-many-retries', freshenv=True)