Merge #152764

152764: cmd/urlcheck: use custom user-agent to appease Wikipedia's WAF r=darrylwong a=srosenberg

It appears that Wikipedia started blocking default user-agents as of 08/25/25 [1]. This correlates with the recent avalanche of failures of `TestNightlyLint`. The attempt to request `https://en.wikipedia.org/wiki/Cron` started to fail with 403,
```
HTTP/1.1 403
Forbidden Connection: close
Content-Length: 92
Content-Type: text/plain
Server: HAProxy
X-Analytics:
X-Cache: cp1100 int
X-Cache-Status: int-tls
```

This change replaces Go's default UA with a custom one. According to Wikipedia's policy [2], we're now in
compliance, which is confirmed by running locally
and getting 200 instead of 403.

[1] https://phabricator.wikimedia.org/T400119
[2] https://foundation.wikimedia.org/wiki/Policy:Wikimedia_Foundation_User-Agent_Policy
Fixes: #152705
Fixes: #152706
Fixes: #152707
Fixes: #152708
Fixes: #152709
Fixes: #152710
Fixes: #152711
Fixes: #152712
Fixes: #152713

Epic: none
Release note: None

Co-authored-by: Stan Rosenberg <[email protected]>

Author: craig[bot]

pkg/cmd/urlcheck/lib/urlcheck/urlcheck.go

@@ -104,7 +104,7 @@ func chompUnbalanced(left, right rune, s string) string {
 }
 
 func checkURL(client *http.Client, url string) error {
-	resp, err := client.Head(url)
+	resp, err := httpDo(client, "HEAD", url)
 	if err != nil {
 		return err
 	}
@@ -121,7 +121,7 @@ func checkURL(client *http.Client, url string) error {
 	// for any other error. Still, we link to several misconfigured servers that
 	// return 403 Forbidden or 500 Internal Server Error for HEAD requests, but
 	// not for GET requests.
-	resp, err = client.Get(url)
+	resp, err = httpDo(client, "GET", url)
 	if err != nil {
 		return err
 	}
@@ -136,6 +136,21 @@ func checkURL(client *http.Client, url string) error {
 	return errors.Newf("%s", errors.Safe(resp.Status))
 }
 
+// N.B. we set custom User-Agent header to avoid being blocked.
+// E.g., as of 08/25/25, Wikipedia blocks default UAs [1].
+// [1] https://phabricator.wikimedia.org/T400119
+func httpDo(c *http.Client, requestType string, url string) (resp *http.Response, err error) {
+	req, err := http.NewRequest(requestType, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	// This UA seems to comply with Wikipedia's policy [1].
+	// [1] https://foundation.wikimedia.org/wiki/Policy:Wikimedia_Foundation_User-Agent_Policy
+	req.Header.Set("User-Agent", "MyGoApplication/1.0 (https://example.com/myapp; [email protected])")
+
+	return c.Do(req)
+}
+
 func checkURLWithRetries(client *http.Client, url string) error {
 	for i := 0; i < timeoutRetries; i++ {
 		err := checkURL(client, url)