Phase 2: Replace got with fetch in retry/timeout files (#57053)

heiskr · web-flow · commit 71436fa5db4f · 2025-08-18T22:32:38.000Z
diff --git a/src/archives/middleware/archived-enterprise-versions-assets.ts b/src/archives/middleware/archived-enterprise-versions-assets.ts
@@ -1,4 +1,4 @@
-import got from 'got'
+import { fetchWithRetry } from '@/frame/lib/fetch-utils'
 import type { Response, NextFunction } from 'express'
 
 import patterns from '@/frame/lib/patterns'
@@ -85,11 +85,31 @@ export default async function archivedEnterpriseVersionsAssets(
 
   const proxyPath = `https://github.github.com/docs-ghes-${requestedVersion}${assetPath}`
   try {
-    const r = await got(proxyPath)
+    const r = await fetchWithRetry(
+      proxyPath,
+      {},
+      {
+        retries: 0,
+        throwHttpErrors: true,
+      },
+    )
+
+    const body = await r.arrayBuffer()
 
     res.set('accept-ranges', 'bytes')
-    res.set('content-type', r.headers['content-type'])
-    res.set('content-length', r.headers['content-length'])
+    const contentType = r.headers.get('content-type')
+    if (contentType) {
+      // Match got's behavior by adding charset=utf-8 to SVG files
+      if (contentType === 'image/svg+xml') {
+        res.set('content-type', `${contentType}; charset=utf-8`)
+      } else {
+        res.set('content-type', contentType)
+      }
+    }
+    const contentLength = r.headers.get('content-length')
+    if (contentLength) {
+      res.set('content-length', contentLength)
+    }
     res.set('x-is-archived', 'true')
     res.set('x-robots-tag', 'noindex')
 
@@ -98,7 +118,7 @@ export default async function archivedEnterpriseVersionsAssets(
     archivedCacheControl(res)
     setFastlySurrogateKey(res, SURROGATE_ENUMS.MANUAL)
 
-    return res.send(r.body)
+    return res.send(Buffer.from(body))
   } catch (err) {
     // Primarily for the developers working on tests that mock
     // requests. If you don't set up `nock` correctly, you might
diff --git a/src/archives/middleware/archived-enterprise-versions.ts b/src/archives/middleware/archived-enterprise-versions.ts
@@ -1,5 +1,5 @@
 import type { Response, NextFunction } from 'express'
-import got from 'got'
+import { fetchWithRetry } from '@/frame/lib/fetch-utils'
 
 import statsd from '@/observability/lib/statsd'
 import {
@@ -190,33 +190,38 @@ export default async function archivedEnterpriseVersions(
   }
   // Retrieve the page from the archived repo
   const doGet = () =>
-    got(getProxyPath(req.path, requestedVersion), {
-      throwHttpErrors: false,
-      retry: retryConfiguration,
-      timeout: timeoutConfiguration,
-    })
+    fetchWithRetry(
+      getProxyPath(req.path, requestedVersion),
+      {},
+      {
+        retries: retryConfiguration.limit,
+        timeout: timeoutConfiguration.response,
+        throwHttpErrors: false,
+      },
+    )
 
   const statsdTags = [`version:${requestedVersion}`]
   const r = await statsd.asyncTimer(doGet, 'archive_enterprise_proxy', [
     ...statsdTags,
     `path:${req.path}`,
   ])()
 
-  if (r.statusCode === 200) {
+  if (r.status === 200) {
+    const body = await r.text()
     const [, withoutLanguagePath] = splitByLanguage(req.path)
     const isDeveloperPage = withoutLanguagePath?.startsWith(
       `/enterprise/${requestedVersion}/developer`,
     )
     res.set('x-robots-tag', 'noindex')
 
     // make stubbed redirect files (which exist in versions <2.13) redirect with a 301
-    const staticRedirect = r.body.match(patterns.staticRedirect)
+    const staticRedirect = body.match(patterns.staticRedirect)
     if (staticRedirect) {
       cacheAggressively(res)
       return res.redirect(redirectCode, staticRedirect[1])
     }
 
-    res.set('content-type', r.headers['content-type'])
+    res.set('content-type', r.headers.get('content-type') || '')
 
     cacheAggressively(res)
 
@@ -230,7 +235,7 @@ export default async function archivedEnterpriseVersions(
       // `x-host` is a custom header set by Fastly.
       // GLB automatically deletes the `x-forwarded-host` header.
       const host = req.get('x-host') || req.get('x-forwarded-host') || req.get('host')
-      r.body = r.body
+      let modifiedBody = body
         .replaceAll(
           `${OLD_AZURE_BLOB_ENTERPRISE_DIR}/${requestedVersion}/assets/cb-`,
           `${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}/assets/cb-`,
@@ -239,6 +244,8 @@ export default async function archivedEnterpriseVersions(
           `${OLD_AZURE_BLOB_ENTERPRISE_DIR}/${requestedVersion}/`,
           `${req.protocol}://${host}/enterprise-server@${requestedVersion}/`,
         )
+
+      return res.send(modifiedBody)
     }
 
     // Releases 3.1 and lower were previously hosted in the
@@ -247,23 +254,42 @@ export default async function archivedEnterpriseVersions(
     // The image paths all need to be updated to reference the images in the
     // new archived enterprise repo's root assets directory.
     if (versionSatisfiesRange(requestedVersion, `<${firstReleaseStoredInBlobStorage}`)) {
-      r.body = r.body.replaceAll(
+      let modifiedBody = body.replaceAll(
         `${OLD_GITHUB_IMAGES_ENTERPRISE_DIR}/${requestedVersion}`,
         `${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}`,
       )
       if (versionSatisfiesRange(requestedVersion, '<=2.18') && isDeveloperPage) {
-        r.body = r.body.replaceAll(
+        modifiedBody = modifiedBody.replaceAll(
           `${OLD_DEVELOPER_SITE_CONTAINER}/${requestedVersion}`,
           `${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}/developer`,
         )
         // Update all hrefs to add /developer to the path
-        r.body = r.body.replaceAll(
+        modifiedBody = modifiedBody.replaceAll(
           `="/enterprise/${requestedVersion}`,
           `="/enterprise/${requestedVersion}/developer`,
         )
         // The changelog is the only thing remaining on developer.github.com
-        r.body = r.body.replaceAll('href="/changes', 'href="https://developer.github.com/changes')
+        modifiedBody = modifiedBody.replaceAll(
+          'href="/changes',
+          'href="https://developer.github.com/changes',
+        )
       }
+
+      // Continue with remaining replacements
+      modifiedBody = modifiedBody.replaceAll(
+        /="(\.\.\/)*assets/g,
+        `="${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}/assets`,
+      )
+
+      // Fix broken hrefs on the 2.16 landing page
+      if (requestedVersion === '2.16' && req.path === '/en/enterprise/2.16') {
+        modifiedBody = modifiedBody.replaceAll('ref="/en/enterprise', 'ref="/en/enterprise/2.16')
+      }
+
+      // Remove the search results container from the page
+      modifiedBody = modifiedBody.replaceAll('<div id="search-results-container"></div>', '')
+
+      return res.send(modifiedBody)
     }
 
     // In all releases, some assets were incorrectly scraped and contain
@@ -275,21 +301,21 @@ export default async function archivedEnterpriseVersions(
     // We want to update the URLs in the format
     // "../../../../../../assets/" to prefix the assets directory with the
     // new archived enterprise repo URL.
-    r.body = r.body.replaceAll(
+    let modifiedBody = body.replaceAll(
       /="(\.\.\/)*assets/g,
       `="${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}/assets`,
     )
 
     // Fix broken hrefs on the 2.16 landing page
     if (requestedVersion === '2.16' && req.path === '/en/enterprise/2.16') {
-      r.body = r.body.replaceAll('ref="/en/enterprise', 'ref="/en/enterprise/2.16')
+      modifiedBody = modifiedBody.replaceAll('ref="/en/enterprise', 'ref="/en/enterprise/2.16')
     }
 
     // Remove the search results container from the page, which removes a white
     // box that prevents clicking on page links
-    r.body = r.body.replaceAll('<div id="search-results-container"></div>', '')
+    modifiedBody = modifiedBody.replaceAll('<div id="search-results-container"></div>', '')
 
-    return res.send(r.body)
+    return res.send(modifiedBody)
   }
   // In releases 2.13 - 2.17, we lost access to frontmatter redirects
   //  during the archival process. This workaround finds potentially
diff --git a/src/frame/lib/fetch-utils.ts b/src/frame/lib/fetch-utils.ts
@@ -0,0 +1,113 @@
+/**
+ * Utility functions for fetch with retry and timeout functionality
+ * to replace got library functionality
+ */
+
+export interface FetchWithRetryOptions {
+  retries?: number
+  retryDelay?: number
+  timeout?: number
+  throwHttpErrors?: boolean
+}
+
+/**
+ * Default retry delay calculation matching got's behavior:
+ * sleep = 1000 * Math.pow(2, retry - 1) + Math.random() * 100
+ */
+function calculateDefaultDelay(attempt: number): number {
+  return 1000 * Math.pow(2, attempt - 1) + Math.random() * 100
+}
+
+/**
+ * Sleep for a given number of milliseconds
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/**
+ * Fetch with timeout support
+ */
+async function fetchWithTimeout(
+  url: string | URL,
+  init?: RequestInit,
+  timeout?: number,
+): Promise<Response> {
+  if (!timeout) {
+    return fetch(url, init)
+  }
+
+  const controller = new AbortController()
+  const timeoutId = setTimeout(() => controller.abort(), timeout)
+
+  try {
+    const response = await fetch(url, {
+      ...init,
+      signal: controller.signal,
+    })
+    clearTimeout(timeoutId)
+    return response
+  } catch (error) {
+    clearTimeout(timeoutId)
+    if (error instanceof Error && error.name === 'AbortError') {
+      throw new Error(`Request timed out after ${timeout}ms`)
+    }
+    throw error
+  }
+}
+
+/**
+ * Fetch with retry logic matching got's behavior
+ */
+export async function fetchWithRetry(
+  url: string | URL,
+  init?: RequestInit,
+  options: FetchWithRetryOptions = {},
+): Promise<Response> {
+  const { retries = 0, timeout, throwHttpErrors = true } = options
+
+  let lastError: Error | null = null
+
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    try {
+      const response = await fetchWithTimeout(url, init, timeout)
+
+      // Check if we should retry based on status code
+      if (response.status >= 500 && attempt < retries) {
+        lastError = new Error(`HTTP ${response.status}: ${response.statusText}`)
+        const delay = calculateDefaultDelay(attempt + 1)
+        await sleep(delay)
+        continue
+      }
+
+      // If throwHttpErrors is true and status indicates an error
+      if (throwHttpErrors && !response.ok && response.status >= 400) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`)
+      }
+
+      return response
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error))
+
+      // Don't retry on the last attempt
+      if (attempt === retries) {
+        throw lastError
+      }
+
+      // Don't retry on client errors (4xx) unless it's specific ones
+      if (
+        error instanceof Error &&
+        error.message.includes('HTTP 4') &&
+        !error.message.includes('HTTP 429')
+      ) {
+        throw lastError
+      }
+
+      // Calculate delay and wait before retry
+      const delay = calculateDefaultDelay(attempt + 1)
+      await sleep(delay)
+    }
+  }
+
+  throw lastError || new Error('Maximum retries exceeded')
+}
diff --git a/src/frame/lib/get-remote-json.js b/src/frame/lib/get-remote-json.js
@@ -2,7 +2,7 @@ import path from 'path'
 import fs from 'fs'
 import crypto from 'crypto'
 
-import got from 'got'
+import { fetchWithRetry } from './fetch-utils'
 import statsd from '@/observability/lib/statsd'
 
 // The only reason this is exported is for the sake of the unit tests'
@@ -65,23 +65,36 @@ export default async function getRemoteJSON(url, config) {
     }
 
     if (!foundOnDisk) {
-      // got will, by default, follow redirects and it will throw if the ultimate
+      // fetch will, by default, follow redirects and fetchWithRetry will throw if the ultimate
       // response is not a 2xx.
       // But it's possible that the page is a 200 OK but it's just not a JSON
       // page at all. Then we can't assume we can deserialize it.
-      const res = await got(url, config)
-      if (!res.headers['content-type'].startsWith('application/json')) {
-        throw new Error(
-          `Fetching '${url}' resulted in a non-JSON response (${res.headers['content-type']})`,
-        )
+      const retries = config?.retry?.limit || 0
+      const timeout = config?.timeout?.response
+
+      const res = await fetchWithRetry(
+        url,
+        {},
+        {
+          retries,
+          timeout,
+          throwHttpErrors: true,
+        },
+      )
+
+      const contentType = res.headers.get('content-type')
+      if (!contentType || !contentType.startsWith('application/json')) {
+        throw new Error(`Fetching '${url}' resulted in a non-JSON response (${contentType})`)
       }
-      cache.set(cacheKey, JSON.parse(res.body))
+
+      const body = await res.text()
+      cache.set(cacheKey, JSON.parse(body))
 
       // Only write to disk for testing and local review.
       // In production, we never write to disk. Only in-memory.
       if (!inProd) {
         fs.mkdirSync(path.dirname(onDisk), { recursive: true })
-        fs.writeFileSync(onDisk, res.body, 'utf-8')
+        fs.writeFileSync(onDisk, body, 'utf-8')
       }
     }
   }
diff --git a/src/links/scripts/check-github-github-links.ts b/src/links/scripts/check-github-github-links.ts
diff --git a/src/links/scripts/rendered-content-link-checker.ts b/src/links/scripts/rendered-content-link-checker.ts
