ensure redirects are followed for sitemap, robots, other requests converted from fetch()

- add a redirect interceptor to getProxyDispatcher(), with option to disable
- add getDefaultDispatcher() for defaulting to follow redirects
- fixes #954

Author: ikreymer

src/util/browser.ts

@@ -10,7 +10,7 @@ import path from "path";
 import { request } from "undici";
 
 import { formatErr, LogContext, logger } from "./logger.js";
-import { getSafeProxyString } from "./proxy.js";
+import { getDefaultDispatcher, getSafeProxyString } from "./proxy.js";
 import { initStorage, S3StorageSync, UploadResult } from "./storage.js";
 
 import {
@@ -250,7 +250,9 @@ export class Browser {
         "browser",
       );
 
-      const resp = await request(profileRemoteSrc);
+      const resp = await request(profileRemoteSrc, {
+        dispatcher: getDefaultDispatcher(),
+      });
 
       await pipeline(resp.body, fs.createWriteStream(profileLocalSrc));
     } else if (profileRemoteSrc && profileRemoteSrc.startsWith("@")) {

src/util/file_reader.ts

@@ -6,7 +6,7 @@ import util from "util";
 import { exec as execCallback } from "child_process";
 
 import { formatErr, logger } from "./logger.js";
-import { getProxyDispatcher } from "./proxy.js";
+import { getDefaultDispatcher, getProxyDispatcher } from "./proxy.js";
 import { parseRecorderFlowJson } from "./flowbehavior.js";
 
 const exec = util.promisify(execCallback);
@@ -90,7 +90,7 @@ async function writeUrlContentsToFile(
 
   try {
     const res = await request(url, {
-      dispatcher: useProxy ? getProxyDispatcher(url) : undefined,
+      dispatcher: useProxy ? getProxyDispatcher(url) : getDefaultDispatcher(),
     });
     if (res.statusCode !== 200) {
       throw new Error(`Invalid response, status: ${res.statusCode}`);

src/util/proxy.ts

@@ -2,7 +2,7 @@ import net from "net";
 import child_process from "child_process";
 import fs from "fs";
 
-import { Agent, Dispatcher, ProxyAgent } from "undici";
+import { Agent, Dispatcher, interceptors, ProxyAgent } from "undici";
 import yaml from "js-yaml";
 
 import { logger } from "./logger.js";
@@ -22,8 +22,13 @@ const SSH_WAIT_TIMEOUT = 30000;
 type ProxyEntry = {
   proxyUrl: string;
   dispatcher: Dispatcher;
+  redirectDispatcher: Dispatcher;
 };
 
+const defaultDispatcher = new Agent();
+
+const defaultRedirectDispatcher = addRedirectInterceptor(defaultDispatcher);
+
 export type ProxyServerConfig = {
   matchHosts?: Record<string, string>;
   proxies?: Record<
@@ -192,7 +197,7 @@ export async function initSingleProxy(
   detached: boolean,
   sshProxyPrivateKeyFile?: string,
   sshProxyKnownHostsFile?: string,
-): Promise<{ proxyUrl: string; dispatcher: Dispatcher }> {
+): Promise<ProxyEntry> {
   logger.debug("Initing proxy", {
     url: getSafeProxyString(proxyUrl),
     localPort,
@@ -215,18 +220,37 @@ export async function initSingleProxy(
   };
 
   const dispatcher = createDispatcher(proxyUrl, agentOpts);
-  return { proxyUrl, dispatcher };
+  const redirectDispatcher = addRedirectInterceptor(dispatcher);
+  return { proxyUrl, dispatcher, redirectDispatcher };
 }
 
-export function getProxyDispatcher(url: string) {
+export function addRedirectInterceptor(dispatcher: Dispatcher) {
+  // match fetch() max redirects if not doing manual redirects
+  // https://fetch.spec.whatwg.org/#http-redirect-fetch
+  const redirector = interceptors.redirect({ maxRedirections: 20 });
+  return dispatcher.compose(redirector);
+}
+
+export function getProxyDispatcher(url: string, withRedirect = true) {
   // find url match by regex first
-  for (const [rx, { dispatcher }] of proxyMap.entries()) {
+  for (const [rx, { dispatcher, redirectDispatcher }] of proxyMap.entries()) {
     if (rx && url.match(rx)) {
-      return dispatcher;
+      return withRedirect ? redirectDispatcher : dispatcher;
     }
   }
-  // if default proxy set, return default dispatcher, otherwise no dispatcher
-  return defaultProxyEntry ? defaultProxyEntry.dispatcher : undefined;
+
+  // if default proxy set, return dispatcher from default proxy, otherwise a default dispatcher
+  if (defaultProxyEntry) {
+    return withRedirect
+      ? defaultProxyEntry.redirectDispatcher
+      : defaultProxyEntry.dispatcher;
+  } else {
+    return getDefaultDispatcher(withRedirect);
+  }
+}
+
+export function getDefaultDispatcher(withRedirect = true) {
+  return withRedirect ? defaultRedirectDispatcher : defaultDispatcher;
 }
 
 export function createDispatcher(

src/util/recorder.ts

@@ -8,7 +8,7 @@ import {
   isRedirectStatus,
 } from "./reqresp.js";
 
-import { Agent, Dispatcher, interceptors, request } from "undici";
+import { Dispatcher, request } from "undici";
 
 import {
   getCustomRewriter,
@@ -55,8 +55,6 @@ const RW_MIME_TYPES = [
 
 const encoder = new TextEncoder();
 
-const defaultAgent = new Agent();
-
 // =================================================================
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function logNetwork(msg: string, data: any) {
@@ -1828,14 +1826,7 @@ class AsyncFetcher {
 
     const headers = reqresp.getRequestHeadersDict();
 
-    let dispatcher = getProxyDispatcher(url) || defaultAgent;
-
-    if (!this.manualRedirect) {
-      // match fetch() max redirects if not doing manual redirects
-      // https://fetch.spec.whatwg.org/#http-redirect-fetch
-      const redirector = interceptors.redirect({ maxRedirections: 20 });
-      dispatcher = dispatcher.compose(redirector);
-    }
+    let dispatcher = getProxyDispatcher(url, !this.manualRedirect);
 
     dispatcher = dispatcher.compose((dispatch) => {
       return (opts, handler) => {