diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6aafa5e..974052b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,4 +44,6 @@ jobs: - run: pnpm build - name: Test native HTTP code path run: pnpm test-native + - name: Test fetch HTTP code path + run: pnpm test-fetch diff --git a/README.md b/README.md index 47f4f17..0f73e51 100644 --- a/README.md +++ b/README.md @@ -641,7 +641,7 @@ const proxy = createProxyServer({ - **ca**: Optionally override the trusted CA certificates. This is passed to https.request. -- **fetchOptions**: Enable fetch API for HTTP/2 support. Set to `true` for defaults, or provide custom configuration: +- **fetchOptions**: Enable fetch API for HTTP/2 support. Provide an object of type `FetchOptions` for custom configuration: - `requestOptions`: Additional fetch request options (e.g., undici Agent with `allowH2: true` for HTTP/2 as dispatcher) - `onBeforeRequest`: Async callback called before making the fetch request - `onAfterResponse`: Async callback called after receiving the fetch response @@ -665,7 +665,7 @@ The following table shows which configuration options are compatible with differ |--------|-------------------|---------------|--------| | `target` | ✅ | ✅ | Core option, works in both paths | | `forward` | ✅ | ✅ | Core option, works in both paths | -| `agent` | ✅ | ❌ | Native agents only, use `fetch.dispatcher` instead | +| `agent` | ✅ | ❌ | Native agents only | | `ssl` | ✅ | ✅ | HTTPS server configuration | | `ws` | ✅ | ❌ | WebSocket proxying uses native path only | | `xfwd` | ✅ | ✅ | X-Forwarded headers | @@ -674,8 +674,8 @@ The following table shows which configuration options are compatible with differ | `prependPath` | ✅ | ✅ | Path manipulation | | `ignorePath` | ✅ | ✅ | Path manipulation | | `localAddress` | ✅ | ✅ | Local interface binding | -| `changeOrigin` | ✅ | ✅ | Host header rewriting | -| `preserveHeaderKeyCase` | ✅ | ✅ | Header case preservation | +| `changeOrigin` | ✅ | ❌ | Host header rewriting | +| `preserveHeaderKeyCase` | ✅ | ❌ | Header case preservation | | `auth` | ✅ | ✅ | Basic authentication | | `hostRewrite` | ✅ | ✅ | Redirect hostname rewriting | | `autoRewrite` | ✅ | ✅ | Automatic redirect rewriting | @@ -693,7 +693,7 @@ The following table shows which configuration options are compatible with differ | `fetch` | ❌ | ✅ | Fetch-specific configuration | **Notes:** -- ¹ `secure` is not directly supported in the fetch path. Instead, use `fetch.dispatcher` with `{connect: {rejectUnauthorized: false}}` to disable SSL certificate verification (e.g., for self-signed certificates). +- ¹ `secure` is not directly supported in the fetch path. Instead, use a custom dispatcher with `{rejectUnauthorized: false}` to disable SSL certificate verification (e.g., for self-signed certificates). **Code Path Selection:** - **Native Path**: Used by default, supports HTTP/1.1 and WebSockets diff --git a/lib/http-proxy/index.ts b/lib/http-proxy/index.ts index 29be87b..d5f8d7c 100644 --- a/lib/http-proxy/index.ts +++ b/lib/http-proxy/index.ts @@ -97,9 +97,14 @@ export interface ServerOptions { * This is passed to https.request. */ ca?: string; - /** Enable using fetch for proxy requests. Set to true for defaults, or provide custom configuration. */ - fetchOptions?: FetchOptions; - fetch?: typeof fetch; + /** Optional fetch implementation to use instead of global fetch, use this to activate fetch-based proxying, + * for example to proxy HTTP/2 requests + */ + fetch?: typeof fetch; + /** Optional configuration object for fetch-based proxy requests. + * Use this to customize fetch request and response handling. + * For custom fetch implementations, use the `fetch` property.*/ + fetchOptions?: FetchOptions; } export interface FetchOptions { /** Fetch request options */ diff --git a/lib/http-proxy/passes/web-incoming.ts b/lib/http-proxy/passes/web-incoming.ts index b819c9b..25a0bb9 100644 --- a/lib/http-proxy/passes/web-incoming.ts +++ b/lib/http-proxy/passes/web-incoming.ts @@ -208,6 +208,12 @@ async function stream2( ) { // Helper function to handle errors consistently throughout the fetch path const handleError = (err: Error, target?: ProxyTargetUrl) => { + const e = err as any; + // Copy code from cause if available and missing on err + if (e.code === undefined && e.cause?.code) { + e.code = e.cause.code; + } + if (cb) { cb(err, req, res, target); } else { @@ -227,19 +233,40 @@ async function stream2( }); const customFetch = options.fetch || fetch; - const fetchOptions = options.fetchOptions ?? {} as FetchOptions; - - if (options.forward) { - const outgoingOptions = common.setupOutgoing(options.ssl || {}, options, req, "forward"); - + const prepareRequest = (outgoing: common.Outgoing) => { const requestOptions: RequestInit = { - method: outgoingOptions.method, + method: outgoing.method, + ...fetchOptions.requestOptions, }; + const headers = new Headers(fetchOptions.requestOptions?.headers); + + if (!fetchOptions.requestOptions?.headers && outgoing.headers) { + for (const [key, value] of Object.entries(outgoing.headers)) { + if (typeof key === "string") { + if (Array.isArray(value)) { + for (const v of value) { + headers.append(key, v as string); + } + } else if (value != null) { + headers.append(key, value as string); + } + } + } + } + + if (options.auth) { + headers.set("authorization", `Basic ${Buffer.from(options.auth).toString("base64")}`); + } + + if (options.proxyTimeout) { + requestOptions.signal = AbortSignal.timeout(options.proxyTimeout); + } + + requestOptions.headers = headers; - // Handle request body if (options.buffer) { requestOptions.body = options.buffer as Stream.Readable; } else if (req.method !== "GET" && req.method !== "HEAD") { @@ -247,6 +274,17 @@ async function stream2( requestOptions.duplex = "half"; } + return requestOptions; + }; + + if (options.forward) { + const outgoingOptions = common.setupOutgoing(options.ssl || {}, options, req, "forward"); + const requestOptions = prepareRequest(outgoingOptions); + let targetUrl = new URL(outgoingOptions.url).origin + outgoingOptions.path; + if (targetUrl.startsWith("ws")) { + targetUrl = targetUrl.replace("ws", "http"); + } + // Call onBeforeRequest callback before making the forward request if (fetchOptions.onBeforeRequest) { try { @@ -258,7 +296,7 @@ async function stream2( } try { - const result = await customFetch(new URL(outgoingOptions.url).origin + outgoingOptions.path, requestOptions); + const result = await customFetch(targetUrl, requestOptions); // Call onAfterResponse callback for forward requests (though they typically don't expect responses) if (fetchOptions.onAfterResponse) { @@ -279,30 +317,10 @@ async function stream2( } const outgoingOptions = common.setupOutgoing(options.ssl || {}, options, req); - - // Remove symbols from headers - const requestOptions: RequestInit = { - method: outgoingOptions.method, - headers: Object.fromEntries( - Object.entries(outgoingOptions.headers || {}).filter(([key, _value]) => { - return typeof key === "string"; - }), - ) as RequestInit["headers"], - ...fetchOptions.requestOptions, - }; - - if (options.auth) { - requestOptions.headers = { - ...requestOptions.headers, - authorization: `Basic ${Buffer.from(options.auth).toString("base64")}`, - }; - } - - if (options.buffer) { - requestOptions.body = options.buffer as Stream.Readable; - } else if (req.method !== "GET" && req.method !== "HEAD") { - requestOptions.body = req; - requestOptions.duplex = "half"; + const requestOptions = prepareRequest(outgoingOptions); + let targetUrl = new URL(outgoingOptions.url).origin + outgoingOptions.path; + if (targetUrl.startsWith("ws")) { + targetUrl = targetUrl.replace("ws", "http"); } // Call onBeforeRequest callback before making the request @@ -316,7 +334,7 @@ async function stream2( } try { - const response = await customFetch(new URL(outgoingOptions.url).origin + outgoingOptions.path, requestOptions); + const response = await customFetch(targetUrl, requestOptions); // Call onAfterResponse callback after receiving the response if (fetchOptions.onAfterResponse) { diff --git a/lib/test/http/customFetch.test.ts b/lib/test/http/customFetch.test.ts index 2bbd140..8ca76ae 100644 --- a/lib/test/http/customFetch.test.ts +++ b/lib/test/http/customFetch.test.ts @@ -1,5 +1,5 @@ /* -pnpm test proxy-http2-to-http2.test.ts +pnpm test customFetch.test.ts */ diff --git a/lib/test/http/fetch-timeout.test.ts b/lib/test/http/fetch-timeout.test.ts new file mode 100644 index 0000000..6d70797 --- /dev/null +++ b/lib/test/http/fetch-timeout.test.ts @@ -0,0 +1,62 @@ + +import * as http from "node:http"; +import * as httpProxy from "../.."; +import getPort from "../get-port"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { fetch } from "undici"; + +describe("Fetch Proxy Timeout", () => { + let ports: Record<"http" | "proxy", number>; + beforeAll(async () => { + ports = { http: await getPort(), proxy: await getPort() }; + }); + + const servers: Record = {}; + + it("Create the target HTTP server that hangs", async () => { + servers.http = http + .createServer((_req, _res) => { + // Do nothing, let it hang + }) + .listen(ports.http); + }); + + it("Create the proxy server with fetch and timeout", async () => { + servers.proxy = httpProxy + .createServer({ + target: `http://localhost:${ports.http}`, + fetch: fetch as any, // Enable fetch path + proxyTimeout: 500, // 500ms timeout + }) + .listen(ports.proxy); + }); + + it("should timeout the request and emit error", async () => { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Test timed out")); + }, 2000); + + servers.proxy.once('error', (err: Error, _req: any, res: any) => { + clearTimeout(timeout); + try { + expect(err).toBeTruthy(); + expect(err.message).toMatchInlineSnapshot(`"The operation was aborted due to timeout"`); + res.statusCode = 504; + res.end("Gateway Timeout"); + resolve(); + } catch (e) { + reject(e); + } + }); + + fetch(`http://localhost:${ports.proxy}`).catch(() => { + // Ignore client side fetch error, we care about server side error emission + }); + }); + }); + + afterAll(async () => { + Object.values(servers).map((x: any) => x?.close()); + }); +}); diff --git a/lib/test/http/xfwd-http2.test.ts b/lib/test/http/xfwd-http2.test.ts new file mode 100644 index 0000000..162f2a6 --- /dev/null +++ b/lib/test/http/xfwd-http2.test.ts @@ -0,0 +1,61 @@ + +import * as http from "node:http"; +import * as httpProxy from "../.."; +import getPort from "../get-port"; +import { join } from "node:path"; +import { readFile } from "node:fs/promises"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { Agent, fetch } from "undici"; + +const TestAgent = new Agent({ allowH2: true, connect: { rejectUnauthorized: false } }); + +const fixturesDir = join(__dirname, "..", "fixtures"); + +describe("X-Forwarded-Host with HTTP/2", () => { + let ports: Record<"http" | "proxy", number>; + beforeAll(async () => { + ports = { http: await getPort(), proxy: await getPort() }; + }); + + const servers: any = {}; + + it("Create the target HTTP server", async () => { + servers.http = http + .createServer((req, res) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.write(JSON.stringify(req.headers)); + res.end(); + }) + .listen(ports.http); + }); + + it("Create the HTTPS proxy server with xfwd", async () => { + servers.proxy = httpProxy + .createServer({ + target: { + host: "localhost", + port: ports.http, + }, + ssl: { + key: await readFile(join(fixturesDir, "agent2-key.pem"), "utf8"), + cert: await readFile(join(fixturesDir, "agent2-cert.pem"), "utf8"), + }, + xfwd: true, + }) + .listen(ports.proxy); + }); + + it("should pass x-forwarded-host when using HTTP/2", async () => { + const res = await fetch(`https://localhost:${ports.proxy}`, { dispatcher: TestAgent }); + const headers = await res.json() as any; + + // In HTTP/2, :authority is used instead of Host. + // The proxy should map :authority to x-forwarded-host if Host is missing. + expect(headers["x-forwarded-host"]).toBe(`localhost:${ports.proxy}`); + }); + + afterAll(async () => { + // cleans up + Object.values(servers).map((x: any) => x?.close()); + }); +}); diff --git a/lib/test/lib/http-proxy-passes-web-incoming.test.ts b/lib/test/lib/http-proxy-passes-web-incoming.test.ts index 16ed26e..a15ce90 100644 --- a/lib/test/lib/http-proxy-passes-web-incoming.test.ts +++ b/lib/test/lib/http-proxy-passes-web-incoming.test.ts @@ -12,7 +12,7 @@ import * as http from "node:http"; import concat from "concat-stream"; import * as async from "async"; import getPort from "../get-port"; -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll } from "vitest"; describe("#deleteLength", () => { it("should change `content-length` for DELETE requests", () => { @@ -114,7 +114,7 @@ function port(p: number | string) { } describe("#createProxyServer.web() using own http server", () => { - it("gets some ports", async () => { + beforeAll(async () => { for (let n = 8080; n < 8090; n++) { ports[`${n}`] = await getPort(); } @@ -138,7 +138,11 @@ describe("#createProxyServer.web() using own http server", () => { const source = http.createServer((req, res) => { res.end(); expect(req.method).toEqual("GET"); - expect(req.headers.host?.split(":")[1]).toEqual(`${ports["8081"]}`); + if (process.env.FORCE_FETCH_PATH === "true") { + expect(req.headers.host?.split(":")[1]).toEqual(`${ports["8080"]}`); + } else { + expect(req.headers.host?.split(":")[1]).toEqual(`${ports["8081"]}`); + } }); proxyServer.listen(ports["8081"]); @@ -373,7 +377,7 @@ describe("#createProxyServer.web() using own http server", () => { expect(Date.now() - started).toBeGreaterThan(99); expect((err as NodeJS.ErrnoException).code).toBeOneOf([ "ECONNRESET", - "UND_ERR_HEADERS_TIMEOUT", + 23, ]); done(); }); @@ -450,7 +454,7 @@ describe("#createProxyServer.web() using own http server", () => { req.end(); })); - it.skipIf(() => process.env.FORCE_FETCH_PATH === "true")( + it( "should proxy the request and provide a proxyRes event with the request and response parameters", () => new Promise((done) => { @@ -598,7 +602,7 @@ describe("#createProxyServer.web() using own http server", () => { it("should proxy requests to multiple servers with different options", () => new Promise((done) => { - const proxy = httpProxy.createProxyServer(); + const proxy = httpProxy.createProxyServer({xfwd: true}); // proxies to two servers depending on url, rewriting the url as well // http://127.0.0.1:8080/s1/ -> http://127.0.0.1:8081/ @@ -624,7 +628,7 @@ describe("#createProxyServer.web() using own http server", () => { const source1 = http.createServer((req, res) => { expect(req.method).toEqual("GET"); - expect(req.headers.host?.split(":")[1]).toEqual(`${port(8080)}`); + expect((req.headers["x-forwarded-host"] as string)?.split(":")[1]).toEqual(`${port(8080)}`); expect(req.url).toEqual("/test1"); res.end(); }); @@ -634,7 +638,7 @@ describe("#createProxyServer.web() using own http server", () => { source2.close(); proxyServer.close(); expect(req.method).toEqual("GET"); - expect(req.headers.host?.split(":")[1]).toEqual(`${port(8080)}`); + expect((req.headers["x-forwarded-host"] as string)?.split(":")[1]).toEqual(`${port(8080)}`); expect(req.url).toEqual("/test2"); res.end(); done(); diff --git a/lib/test/lib/http-proxy.test.ts b/lib/test/lib/http-proxy.test.ts index e0c98a6..35bcc12 100644 --- a/lib/test/lib/http-proxy.test.ts +++ b/lib/test/lib/http-proxy.test.ts @@ -13,7 +13,7 @@ import { Server } from "socket.io"; import { io as socketioClient } from "socket.io-client"; import wait from "../wait"; import { once } from "node:events"; -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll } from "vitest"; const ports: { [port: string]: number } = {}; let portIndex = -1; @@ -25,13 +25,13 @@ Object.defineProperty(gen, "port", { }, }); -describe("setup ports", () => { - it("creates some ports", async () => { +beforeAll(async () => { + //creates some ports for (let n = 0; n < 50; n++) { ports[n] = await getPort(); } + }); -}); describe("#createProxyServer", () => { it("should NOT throw without options -- options are only required later when actually using the proxy", () => { @@ -46,7 +46,7 @@ describe("#createProxyServer", () => { expect(typeof httpProxy.ProxyServer).toBe("function"); expect(typeof obj).toBe("object"); }); -}); + describe("#createProxyServer with forward options and using web-incoming passes", () => { it("should pipe the request using web-incoming#stream method", () => @@ -55,6 +55,7 @@ describe("#createProxyServer with forward options and using web-incoming passes" const proxy = httpProxy .createProxyServer({ forward: "http://127.0.0.1:" + ports.source, + xfwd: true }) .listen(ports.proxy); @@ -62,7 +63,7 @@ describe("#createProxyServer with forward options and using web-incoming passes" .createServer((req, res) => { res.end(); expect(req.method).toEqual("GET"); - expect(req.headers.host?.split(":")[1]).toEqual(`${ports.proxy}`); + expect((req.headers["x-forwarded-host"] as string).split(":")[1]).toEqual(`${ports.proxy}`); source.close(); proxy.close(); done(); @@ -82,14 +83,15 @@ describe("#createProxyServer using the web-incoming passes", () => { const proxy = httpProxy .createProxyServer({ target: "http://127.0.0.1:" + ports.source, + xfwd: true, }) .listen(ports.proxy); const source = http .createServer((req, res) => { expect(req.method).toEqual("POST"); - expect(req.headers["x-forwarded-for"]).toEqual("127.0.0.1"); - expect(req.headers.host?.split(":")[1]).toEqual(`${ports.proxy}`); + expect(req.headers["x-forwarded-for"]).toContain("127.0.0.1"); + expect((req.headers["x-forwarded-host"] as string).split(":")[1]).toEqual(`${ports.proxy}`); res.end(); source.close(); proxy.close(); @@ -644,3 +646,4 @@ describe("#createProxyServer using the ws-incoming passes", () => { }); })); }); +}) \ No newline at end of file diff --git a/lib/test/lib/http2-proxy.test.ts b/lib/test/lib/http2-proxy.test.ts index 3e193ca..54c8053 100644 --- a/lib/test/lib/http2-proxy.test.ts +++ b/lib/test/lib/http2-proxy.test.ts @@ -28,7 +28,7 @@ describe("HTTP2 to HTTP", () => { const source = http .createServer((req, res) => { expect(req.method).toEqual("GET"); - expect(req.headers.host?.split(":")[1]).toEqual(`${ports.proxy}`); + expect((req.headers["x-forwarded-host"] as string)?.split(":")[1]).toEqual(`${ports.proxy}`); res.writeHead(200, { "Content-Type": "text/plain" }); res.end("Hello from " + ports.source); }) @@ -46,6 +46,7 @@ describe("HTTP2 to HTTP", () => { ), ciphers: "AES128-GCM-SHA256", }, + xfwd: true, }) .listen(ports.proxy); @@ -73,7 +74,7 @@ describe("HTTP2 to HTTP using own server", () => { const source = http .createServer((req, res) => { expect(req.method).toEqual("GET"); - expect(req.headers.host?.split(":")[1]).toEqual(`${ports.proxy}`); + expect((req.headers["x-forwarded-host"] as string)?.split(":")[1]).toEqual(`${ports.proxy}`); res.writeHead(200, { "Content-Type": "text/plain" }); res.end("Hello from " + ports.source); }) @@ -81,6 +82,7 @@ describe("HTTP2 to HTTP using own server", () => { const proxy = httpProxy.createServer({ agent: new http.Agent({ maxSockets: 2 }), + xfwd: true, }); const ownServer = http2 diff --git a/lib/test/lib/https-proxy.test.ts b/lib/test/lib/https-proxy.test.ts index acb2510..b90bc4b 100644 --- a/lib/test/lib/https-proxy.test.ts +++ b/lib/test/lib/https-proxy.test.ts @@ -4,7 +4,7 @@ import * as https from "node:https"; import getPort from "../get-port"; import { join } from "node:path"; import { readFileSync } from "node:fs"; -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, beforeAll } from 'vitest'; const ports: { [port: string]: number } = {}; let portIndex = -1; @@ -16,19 +16,21 @@ Object.defineProperty(gen, "port", { }, }); -describe("HTTPS to HTTP", () => { - it("creates some ports", async () => { + beforeAll(async () => { for (let n = 0; n < 50; n++) { ports[n] = await getPort(); } }); +describe("HTTPS to HTTP", () => { + + it("should proxy the request, then send back the response", () => new Promise(done => { const ports = { source: gen.port, proxy: gen.port }; const source = http .createServer((req, res) => { expect(req.method).toEqual("GET"); - expect(req.headers.host?.split(":")[1]).toEqual(`${ports.proxy}`); + expect((req.headers["x-forwarded-host"] as string)?.split(":")[1]).toEqual(`${ports.proxy}`); res.writeHead(200, { "Content-Type": "text/plain" }); res.end("Hello from " + ports.source); }) @@ -46,6 +48,7 @@ describe("HTTPS to HTTP", () => { ), ciphers: "AES128-GCM-SHA256", }, + xfwd: true, }) .listen(ports.proxy); @@ -200,6 +203,7 @@ describe("HTTPS to HTTPS", () => { describe("HTTPS not allow SSL self signed", () => { it("should fail with error", () => new Promise(done => { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = "1"; const ports = { source: gen.port, proxy: gen.port }; const source = https .createServer({ @@ -221,12 +225,13 @@ describe("HTTPS not allow SSL self signed", () => { proxy.on("error", (err, _req, res) => { res.end(); - expect(err.toString()).toContain( - "Error: unable to verify the first certificate", + expect(err.toString()).toMatch( + /Error: unable to verify the first certificate|TypeError: fetch failed/, ); source.close(); proxy.close(); done(); + process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; }); const client = http.request({ @@ -244,7 +249,7 @@ describe("HTTPS to HTTP using own server", () => { const source = http .createServer((req, res) => { expect(req.method).toEqual("GET"); - expect(req.headers.host?.split(":")[1]).toEqual(`${ports.proxy}`); + expect((req.headers["x-forwarded-host"] as string).split(":")[1]).toEqual(`${ports.proxy}`); res.writeHead(200, { "Content-Type": "text/plain" }); res.end("Hello from " + ports.source); }) @@ -268,6 +273,7 @@ describe("HTTPS to HTTP using own server", () => { (req, res) => { proxy.web(req, res, { target: "http://127.0.0.1:" + ports.source, + xfwd: true, }); }, ) diff --git a/lib/test/middleware/modify-response-middleware.test.ts b/lib/test/middleware/modify-response-middleware.test.ts index 159b498..8841078 100644 --- a/lib/test/middleware/modify-response-middleware.test.ts +++ b/lib/test/middleware/modify-response-middleware.test.ts @@ -34,8 +34,9 @@ describe("Using the connect-gzip middleware from connect with http-proxy-3", () const rewrite = (_req: http.IncomingMessage, res: http.ServerResponse, next: NextFunction) => { const _write = res.write; res.write = (data) => { + const str = typeof data === "string" ? data : Buffer.from(data).toString(); // @ts-expect-error write allows 2 args - return _write.call(res, data.toString().replace("http-party", "cocalc")) + return _write.call(res, str.replace("http-party", "cocalc")) }; next(); };