From 57c4ab1956d60cc83a2ab2589468c8b9f4cb55ce Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 18 Apr 2025 10:04:45 +0200 Subject: [PATCH 1/7] wip: proxy agents --- demos/example-node/src/main.ts | 4 +- .../src/client/sync/stream/AbstractRemote.ts | 14 +++- packages/node/package.json | 5 +- packages/node/src/sync/stream/NodeRemote.ts | 18 ++++- pnpm-lock.yaml | 76 +++++++++---------- 5 files changed, 71 insertions(+), 46 deletions(-) diff --git a/demos/example-node/src/main.ts b/demos/example-node/src/main.ts index 6dc6e777f..cf6ac280a 100644 --- a/demos/example-node/src/main.ts +++ b/demos/example-node/src/main.ts @@ -1,10 +1,10 @@ -import repl_factory from 'node:repl'; import { once } from 'node:events'; +import repl_factory from 'node:repl'; import { PowerSyncDatabase, SyncStreamConnectionMethod } from '@powersync/node'; import { default as Logger } from 'js-logger'; -import { AppSchema, DemoConnector } from './powersync.js'; import { exit } from 'node:process'; +import { AppSchema, DemoConnector } from './powersync.js'; const main = async () => { const logger = Logger.get('PowerSyncDemo'); diff --git a/packages/common/src/client/sync/stream/AbstractRemote.ts b/packages/common/src/client/sync/stream/AbstractRemote.ts index d0b6ab611..8446762dd 100644 --- a/packages/common/src/client/sync/stream/AbstractRemote.ts +++ b/packages/common/src/client/sync/stream/AbstractRemote.ts @@ -85,6 +85,9 @@ export type AbstractRemoteOptions = { * Binding should be done before passing here. */ fetchImplementation: FetchImplementation | FetchImplementationProvider; + + // TODO + fetchOptions?: {}; }; export const DEFAULT_REMOTE_OPTIONS: AbstractRemoteOptions = { @@ -92,7 +95,8 @@ export const DEFAULT_REMOTE_OPTIONS: AbstractRemoteOptions = { url.replace(/^https?:\/\//, function (match) { return match === 'https://' ? 'wss://' : 'ws://'; }), - fetchImplementation: new FetchImplementationProvider() + fetchImplementation: new FetchImplementationProvider(), + fetchOptions: {} }; export abstract class AbstractRemote { @@ -231,6 +235,10 @@ export abstract class AbstractRemote { */ abstract getBSON(): Promise; + protected createSocket(url: string): WebSocket { + return new WebSocket(url); + } + /** * Connects to the sync/stream websocket endpoint */ @@ -249,7 +257,8 @@ export abstract class AbstractRemote { const connector = new RSocketConnector({ transport: new WebsocketClientTransport({ - url: this.options.socketUrlTransformer(request.url) + url: this.options.socketUrlTransformer(request.url), + wsCreator: (url) => this.createSocket(url) }), setup: { keepAlive: KEEP_ALIVE_MS, @@ -421,6 +430,7 @@ export abstract class AbstractRemote { body: JSON.stringify(data), signal: controller.signal, cache: 'no-store', + ...(this.options.fetchOptions ?? {}), ...options.fetchOptions }).catch((ex) => { if (ex.name == 'AbortError') { diff --git a/packages/node/package.json b/packages/node/package.json index c1eee68fc..6f4ca7ca2 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -51,12 +51,13 @@ "@powersync/common": "workspace:*", "async-lock": "^1.4.0", "bson": "^6.6.0", - "comlink": "^4.4.2" + "comlink": "^4.4.2", + "undici": "^7.8.0" }, "devDependencies": { + "@powersync/drizzle-driver": "workspace:*", "@types/async-lock": "^1.4.0", "drizzle-orm": "^0.35.2", - "@powersync/drizzle-driver": "workspace:*", "rollup": "4.14.3", "typescript": "^5.5.3", "vitest": "^3.0.5" diff --git a/packages/node/src/sync/stream/NodeRemote.ts b/packages/node/src/sync/stream/NodeRemote.ts index b0c147484..204ba375e 100644 --- a/packages/node/src/sync/stream/NodeRemote.ts +++ b/packages/node/src/sync/stream/NodeRemote.ts @@ -12,6 +12,7 @@ import { RemoteConnector } from '@powersync/common'; import { BSON } from 'bson'; +import { ProxyAgent } from 'undici'; export const STREAMING_POST_TIMEOUT_MS = 30_000; @@ -27,12 +28,27 @@ export class NodeRemote extends AbstractRemote { protected logger: ILogger = DEFAULT_REMOTE_LOGGER, options?: Partial ) { + // Automatic env vars are not supported by undici + // proxy-agent does not work directly with dispatcher + const proxy = process.env.HTTPS_PROXY ?? process.env.HTTP_PROXY; super(connector, logger, { ...(options ?? {}), - fetchImplementation: options?.fetchImplementation ?? new NodeFetchProvider() + fetchImplementation: options?.fetchImplementation ?? new NodeFetchProvider(), + fetchOptions: { + dispatcher: proxy ? new ProxyAgent(proxy) : undefined + } }); } + // protected createSocket(url: string): globalThis.WebSocket { + // return new WebSocket(url, { + // // agent: new ProxyAgent(), + // headers: { + // 'User-Agent': this.getUserAgent() + // } + // }) as any as globalThis.WebSocket; + // } + getUserAgent(): string { return [ super.getUserAgent(), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8814f331b..96c4b2f87 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1807,6 +1807,9 @@ importers: comlink: specifier: ^4.4.2 version: 4.4.2 + undici: + specifier: ^7.8.0 + version: 7.8.0 devDependencies: '@powersync/drizzle-driver': specifier: workspace:* @@ -9685,10 +9688,6 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} - agent-base@7.1.1: - resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==} - engines: {node: '>= 14'} - agent-base@7.1.3: resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==} engines: {node: '>= 14'} @@ -16145,8 +16144,8 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} - pac-proxy-agent@7.0.2: - resolution: {integrity: sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==} + pac-proxy-agent@7.2.0: + resolution: {integrity: sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==} engines: {node: '>= 14'} pac-resolver@7.0.1: @@ -17079,8 +17078,8 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} - proxy-agent@6.4.0: - resolution: {integrity: sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==} + proxy-agent@6.5.0: + resolution: {integrity: sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==} engines: {node: '>= 14'} proxy-from-env@1.1.0: @@ -18415,6 +18414,10 @@ packages: resolution: {integrity: sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==} engines: {node: '>= 14'} + socks-proxy-agent@8.0.5: + resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} + engines: {node: '>= 14'} + socks@2.8.3: resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} @@ -19404,6 +19407,10 @@ packages: resolution: {integrity: sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==} engines: {node: '>=18.17'} + undici@7.8.0: + resolution: {integrity: sha512-vFv1GA99b7eKO1HG/4RPu2Is3FBTWBrmzqzO0mz+rLxN3yXkE4mqRcb8g8fHxzX4blEysrNZLqg5RbJLqX5buA==} + engines: {node: '>=20.18.1'} + unicode-canonical-property-names-ecmascript@2.0.1: resolution: {integrity: sha512-dA8WbNeb2a6oQzAQ55YlT5vQAWGV9WXOsi3SskE3bcCdM0P4SDd+24zS/OCacdRq5BkdsRj9q3Pg6YyQoxIGqg==} engines: {node: '>=4'} @@ -20387,18 +20394,6 @@ packages: utf-8-validate: optional: true - ws@8.18.0: - resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - ws@8.18.1: resolution: {integrity: sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==} engines: {node: '>=10.0.0'} @@ -26790,7 +26785,7 @@ snapshots: '@npmcli/agent@3.0.0': dependencies: - agent-base: 7.1.1 + agent-base: 7.1.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 10.4.3 @@ -27331,7 +27326,7 @@ snapshots: debug: 4.4.0(supports-color@8.1.1) extract-zip: 2.0.1 progress: 2.0.3 - proxy-agent: 6.4.0 + proxy-agent: 6.5.0 semver: 7.7.1 tar-fs: 3.0.6 unbzip2-stream: 1.4.3 @@ -32319,12 +32314,6 @@ snapshots: transitivePeerDependencies: - supports-color - agent-base@7.1.1: - dependencies: - debug: 4.4.0(supports-color@8.1.1) - transitivePeerDependencies: - - supports-color - agent-base@7.1.3: {} agentkeepalive@4.5.0: @@ -37566,7 +37555,7 @@ snapshots: http-proxy-agent@7.0.2: dependencies: - agent-base: 7.1.1 + agent-base: 7.1.3 debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -37640,7 +37629,7 @@ snapshots: https-proxy-agent@7.0.5: dependencies: - agent-base: 7.1.1 + agent-base: 7.1.3 debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -38944,7 +38933,7 @@ snapshots: whatwg-encoding: 3.1.1 whatwg-mimetype: 4.0.0 whatwg-url: 14.0.0 - ws: 8.18.0 + ws: 8.18.1 xml-name-validator: 5.0.0 transitivePeerDependencies: - bufferutil @@ -41585,7 +41574,7 @@ snapshots: p-try@2.2.0: {} - pac-proxy-agent@7.0.2: + pac-proxy-agent@7.2.0: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.3 @@ -41594,7 +41583,7 @@ snapshots: http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 pac-resolver: 7.0.1 - socks-proxy-agent: 8.0.4 + socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color optional: true @@ -42647,16 +42636,16 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 - proxy-agent@6.4.0: + proxy-agent@6.5.0: dependencies: agent-base: 7.1.3 debug: 4.4.0(supports-color@8.1.1) http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 7.18.3 - pac-proxy-agent: 7.0.2 + pac-proxy-agent: 7.2.0 proxy-from-env: 1.1.0 - socks-proxy-agent: 8.0.4 + socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color optional: true @@ -44774,12 +44763,21 @@ snapshots: socks-proxy-agent@8.0.4: dependencies: - agent-base: 7.1.1 + agent-base: 7.1.3 debug: 4.4.0(supports-color@8.1.1) socks: 2.8.3 transitivePeerDependencies: - supports-color + socks-proxy-agent@8.0.5: + dependencies: + agent-base: 7.1.3 + debug: 4.4.0(supports-color@8.1.1) + socks: 2.8.3 + transitivePeerDependencies: + - supports-color + optional: true + socks@2.8.3: dependencies: ip-address: 9.0.5 @@ -46044,6 +46042,8 @@ snapshots: undici@6.21.0: {} + undici@7.8.0: {} + unicode-canonical-property-names-ecmascript@2.0.1: {} unicode-emoji-modifier-base@1.0.0: {} @@ -47439,8 +47439,6 @@ snapshots: ws@7.5.10: {} - ws@8.18.0: {} - ws@8.18.1: {} xcode@3.0.1: From 468dc580012f325acbf6b7072d2d099dacb955a8 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 18 Apr 2025 10:33:50 +0200 Subject: [PATCH 2/7] proxy for websockets --- packages/node/package.json | 4 +- packages/node/src/sync/stream/NodeRemote.ts | 27 ++++++++----- pnpm-lock.yaml | 44 +++++++-------------- 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/packages/node/package.json b/packages/node/package.json index 6f4ca7ca2..223a6a08a 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -52,7 +52,9 @@ "async-lock": "^1.4.0", "bson": "^6.6.0", "comlink": "^4.4.2", - "undici": "^7.8.0" + "proxy-agent": "^6.5.0", + "undici": "^7.8.0", + "ws": "^8.18.1" }, "devDependencies": { "@powersync/drizzle-driver": "workspace:*", diff --git a/packages/node/src/sync/stream/NodeRemote.ts b/packages/node/src/sync/stream/NodeRemote.ts index 204ba375e..62b1a568b 100644 --- a/packages/node/src/sync/stream/NodeRemote.ts +++ b/packages/node/src/sync/stream/NodeRemote.ts @@ -12,7 +12,9 @@ import { RemoteConnector } from '@powersync/common'; import { BSON } from 'bson'; +import Agent from 'proxy-agent'; import { ProxyAgent } from 'undici'; +import { WebSocket } from 'ws'; export const STREAMING_POST_TIMEOUT_MS = 30_000; @@ -23,6 +25,8 @@ class NodeFetchProvider extends FetchImplementationProvider { } export class NodeRemote extends AbstractRemote { + protected agent: ProxyAgent | undefined; + constructor( protected connector: RemoteConnector, protected logger: ILogger = DEFAULT_REMOTE_LOGGER, @@ -31,23 +35,28 @@ export class NodeRemote extends AbstractRemote { // Automatic env vars are not supported by undici // proxy-agent does not work directly with dispatcher const proxy = process.env.HTTPS_PROXY ?? process.env.HTTP_PROXY; + const agent = proxy ? new ProxyAgent(proxy) : undefined; + super(connector, logger, { ...(options ?? {}), fetchImplementation: options?.fetchImplementation ?? new NodeFetchProvider(), fetchOptions: { - dispatcher: proxy ? new ProxyAgent(proxy) : undefined + dispatcher: agent } }); + + this.agent = agent; } - // protected createSocket(url: string): globalThis.WebSocket { - // return new WebSocket(url, { - // // agent: new ProxyAgent(), - // headers: { - // 'User-Agent': this.getUserAgent() - // } - // }) as any as globalThis.WebSocket; - // } + protected createSocket(url: string): globalThis.WebSocket { + return new WebSocket(url, { + // Undici does not seem to be compatible with ws + agent: new Agent.ProxyAgent(), + headers: { + 'User-Agent': this.getUserAgent() + } + }) as any as globalThis.WebSocket; // TODO + } getUserAgent(): string { return [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96c4b2f87..8bc5dfd0c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1807,9 +1807,15 @@ importers: comlink: specifier: ^4.4.2 version: 4.4.2 + proxy-agent: + specifier: ^6.5.0 + version: 6.5.0 undici: specifier: ^7.8.0 version: 7.8.0 + ws: + specifier: ^8.18.1 + version: 8.18.1 devDependencies: '@powersync/drizzle-driver': specifier: workspace:* @@ -18410,10 +18416,6 @@ packages: resolution: {integrity: sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==} engines: {node: '>= 10'} - socks-proxy-agent@8.0.4: - resolution: {integrity: sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==} - engines: {node: '>= 14'} - socks-proxy-agent@8.0.5: resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} engines: {node: '>= 14'} @@ -26789,7 +26791,7 @@ snapshots: http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 10.4.3 - socks-proxy-agent: 8.0.4 + socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color @@ -28768,7 +28770,9 @@ snapshots: transitivePeerDependencies: - '@babel/core' - '@babel/preset-env' + - bufferutil - supports-color + - utf-8-validate '@react-native/normalize-color@2.1.0': {} @@ -31005,8 +31009,7 @@ snapshots: '@tootallnate/once@2.0.0': {} - '@tootallnate/quickjs-emscripten@0.23.0': - optional: true + '@tootallnate/quickjs-emscripten@0.23.0': {} '@trysound/sax@0.2.0': {} @@ -32634,7 +32637,6 @@ snapshots: ast-types@0.13.4: dependencies: tslib: 2.8.1 - optional: true ast-types@0.15.2: dependencies: @@ -32993,8 +32995,7 @@ snapshots: dependencies: safe-buffer: 5.1.2 - basic-ftp@5.0.5: - optional: true + basic-ftp@5.0.5: {} batch@0.6.1: {} @@ -34317,8 +34318,7 @@ snapshots: data-uri-to-buffer@4.0.1: {} - data-uri-to-buffer@6.0.2: - optional: true + data-uri-to-buffer@6.0.2: {} data-urls@3.0.2: dependencies: @@ -34490,7 +34490,6 @@ snapshots: ast-types: 0.13.4 escodegen: 2.1.0 esprima: 4.0.1 - optional: true del-cli@5.1.0: dependencies: @@ -36993,7 +36992,6 @@ snapshots: fs-extra: 11.2.0 transitivePeerDependencies: - supports-color - optional: true getenv@1.0.0: {} @@ -41044,8 +41042,7 @@ snapshots: nested-error-stacks@2.0.1: {} - netmask@2.0.2: - optional: true + netmask@2.0.2: {} next@14.2.3(@babel/core@7.26.10)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(sass@1.79.4): dependencies: @@ -41586,13 +41583,11 @@ snapshots: socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color - optional: true pac-resolver@7.0.1: dependencies: degenerator: 5.0.1 netmask: 2.0.2 - optional: true package-json-from-dist@1.0.1: {} @@ -42648,10 +42643,8 @@ snapshots: socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color - optional: true - proxy-from-env@1.1.0: - optional: true + proxy-from-env@1.1.0: {} prr@1.0.1: optional: true @@ -44761,14 +44754,6 @@ snapshots: transitivePeerDependencies: - supports-color - socks-proxy-agent@8.0.4: - dependencies: - agent-base: 7.1.3 - debug: 4.4.0(supports-color@8.1.1) - socks: 2.8.3 - transitivePeerDependencies: - - supports-color - socks-proxy-agent@8.0.5: dependencies: agent-base: 7.1.3 @@ -44776,7 +44761,6 @@ snapshots: socks: 2.8.3 transitivePeerDependencies: - supports-color - optional: true socks@2.8.3: dependencies: From 9df301864dbe3687ef1e38956d3082a80f3f03ec Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Fri, 18 Apr 2025 13:52:43 +0200 Subject: [PATCH 3/7] Added minor documentation and typing. --- .changeset/hip-lamps-draw.md | 5 +++++ .changeset/thick-lies-invent.md | 5 +++++ .../src/client/sync/stream/AbstractRemote.ts | 7 ++++++- packages/node/README.md | 16 ++++++++++++++++ packages/node/src/sync/stream/NodeRemote.ts | 5 +++-- 5 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 .changeset/hip-lamps-draw.md create mode 100644 .changeset/thick-lies-invent.md diff --git a/.changeset/hip-lamps-draw.md b/.changeset/hip-lamps-draw.md new file mode 100644 index 000000000..2224c6d88 --- /dev/null +++ b/.changeset/hip-lamps-draw.md @@ -0,0 +1,5 @@ +--- +'@powersync/node': minor +--- + +Introduced support for specifying proxy environment variables for the connection methods. For HTTP it supports `HTTP_PROXY` or `HTTPS_PROXY`, and for WebSockets it supports `WS_PROXY` and `WSS_PROXY`. diff --git a/.changeset/thick-lies-invent.md b/.changeset/thick-lies-invent.md new file mode 100644 index 000000000..f3480ddcf --- /dev/null +++ b/.changeset/thick-lies-invent.md @@ -0,0 +1,5 @@ +--- +'@powersync/common': minor +--- + +Added `fetchOptions` to AbstractRemoteOptions. Allows consumers to include fields such as `dispatcher` (e.g. for proxy support) to the fetch invocations. diff --git a/packages/common/src/client/sync/stream/AbstractRemote.ts b/packages/common/src/client/sync/stream/AbstractRemote.ts index 8446762dd..79907443b 100644 --- a/packages/common/src/client/sync/stream/AbstractRemote.ts +++ b/packages/common/src/client/sync/stream/AbstractRemote.ts @@ -86,7 +86,12 @@ export type AbstractRemoteOptions = { */ fetchImplementation: FetchImplementation | FetchImplementationProvider; - // TODO + /** + * Optional options to pass directly to all `fetch` calls. + * + * This can include fields such as `dispatcher` (e.g. for proxy support), + * `cache`, or any other fetch-compatible options. + */ fetchOptions?: {}; }; diff --git a/packages/node/README.md b/packages/node/README.md index c64e2cc78..e06b9d3ff 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -56,6 +56,22 @@ contains everything you need to know to get started implementing PowerSync in yo A simple example using `@powersync/node` is available in the [`demos/example-node/`](../demos/example-node) directory. +# Proxy Support + +This SDK supports HTTP, HTTPS, and WebSocket proxies via environment variables. + +## HTTP Connection Method + +Internally we probe the http environment variables and apply it to fetch requests ([undici](https://www.npmjs.com/package/undici/v/5.6.0)) + +- Set the `HTTPS_PROXY` or `HTTP_PROXY` environment variable to automatically route HTTP requests through a proxy. + +## WEB Socket Connection Method + +Internally the [proxy-agent](https://www.npmjs.com/package/proxy-agent) dependency for WebSocket proxies, which has its own internal code for automatically picking up the appropriate environment variables: + +- Set the `WS_PROXY` or `WSS_PROXY` environment variable to route the webocket connections through a proxy. + # Found a bug or need help? - Join our [Discord server](https://discord.gg/powersync) where you can browse topics from our community, ask questions, share feedback, or just say hello :) diff --git a/packages/node/src/sync/stream/NodeRemote.ts b/packages/node/src/sync/stream/NodeRemote.ts index 62b1a568b..1dc470ecf 100644 --- a/packages/node/src/sync/stream/NodeRemote.ts +++ b/packages/node/src/sync/stream/NodeRemote.ts @@ -50,12 +50,13 @@ export class NodeRemote extends AbstractRemote { protected createSocket(url: string): globalThis.WebSocket { return new WebSocket(url, { - // Undici does not seem to be compatible with ws + // Undici does not seem to be compatible with ws, using `proxy-agent` instead. + // Automatically uses WS_PROXY or WSS_PROXY env vars agent: new Agent.ProxyAgent(), headers: { 'User-Agent': this.getUserAgent() } - }) as any as globalThis.WebSocket; // TODO + }) as any as globalThis.WebSocket; // This is compatible in Node environments } getUserAgent(): string { From e2508cedad2550b10c692edc6c1d79b277486242 Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Tue, 22 Apr 2025 19:53:33 +0200 Subject: [PATCH 4/7] Using EnvHttpProxyAgent for HTTP proxy to automatically pick up envs. --- packages/node/src/sync/stream/NodeRemote.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/node/src/sync/stream/NodeRemote.ts b/packages/node/src/sync/stream/NodeRemote.ts index 1dc470ecf..1384d9b1a 100644 --- a/packages/node/src/sync/stream/NodeRemote.ts +++ b/packages/node/src/sync/stream/NodeRemote.ts @@ -13,7 +13,7 @@ import { } from '@powersync/common'; import { BSON } from 'bson'; import Agent from 'proxy-agent'; -import { ProxyAgent } from 'undici'; +import { ProxyAgent, EnvHttpProxyAgent } from 'undici'; import { WebSocket } from 'ws'; export const STREAMING_POST_TIMEOUT_MS = 30_000; @@ -32,10 +32,8 @@ export class NodeRemote extends AbstractRemote { protected logger: ILogger = DEFAULT_REMOTE_LOGGER, options?: Partial ) { - // Automatic env vars are not supported by undici - // proxy-agent does not work directly with dispatcher - const proxy = process.env.HTTPS_PROXY ?? process.env.HTTP_PROXY; - const agent = proxy ? new ProxyAgent(proxy) : undefined; + // Automatically uses relevant env vars for HTTP + const agent = new EnvHttpProxyAgent(); super(connector, logger, { ...(options ?? {}), @@ -50,8 +48,7 @@ export class NodeRemote extends AbstractRemote { protected createSocket(url: string): globalThis.WebSocket { return new WebSocket(url, { - // Undici does not seem to be compatible with ws, using `proxy-agent` instead. - // Automatically uses WS_PROXY or WSS_PROXY env vars + // Automatically uses relevant env vars for web sockets agent: new Agent.ProxyAgent(), headers: { 'User-Agent': this.getUserAgent() From 43c08b82b545b25873d3e380ee2eed79e9e2bb55 Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Wed, 23 Apr 2025 09:38:24 +0200 Subject: [PATCH 5/7] Added dispatcher override config option. --- .changeset/thirty-humans-shop.md | 5 +++ .../src/client/AbstractPowerSyncDatabase.ts | 38 +++++++++---------- packages/node/src/db/PowerSyncDatabase.ts | 22 ++++++++++- packages/node/src/sync/stream/NodeRemote.ts | 18 ++++----- 4 files changed, 52 insertions(+), 31 deletions(-) create mode 100644 .changeset/thirty-humans-shop.md diff --git a/.changeset/thirty-humans-shop.md b/.changeset/thirty-humans-shop.md new file mode 100644 index 000000000..253f68c9e --- /dev/null +++ b/.changeset/thirty-humans-shop.md @@ -0,0 +1,5 @@ +--- +'@powersync/common': patch +--- + +Ensuring all options provided to `connect()` are passed onwards, allows packages to have their own option definitions for `connect()` and the abstract `generateSyncStreamImplementation()`. diff --git a/packages/common/src/client/AbstractPowerSyncDatabase.ts b/packages/common/src/client/AbstractPowerSyncDatabase.ts index 2e71083e7..036decb3e 100644 --- a/packages/common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/common/src/client/AbstractPowerSyncDatabase.ts @@ -417,6 +417,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { this.currentStatus = new SyncStatus({ @@ -555,7 +553,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { @@ -633,7 +631,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { @@ -661,7 +659,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { + return super.connect(connector, options); + } + protected generateSyncStreamImplementation( - connector: PowerSyncBackendConnector + connector: PowerSyncBackendConnector, + options: NodeAdditionalConnectionOptions ): AbstractStreamingSyncImplementation { - const remote = new NodeRemote(connector); + const remote = new NodeRemote(connector, this.options.logger, { dispatcher: options.dispatcher }); return new NodeStreamingSyncImplementation({ adapter: this.bucketStorageAdapter, diff --git a/packages/node/src/sync/stream/NodeRemote.ts b/packages/node/src/sync/stream/NodeRemote.ts index 1384d9b1a..5e76e0f09 100644 --- a/packages/node/src/sync/stream/NodeRemote.ts +++ b/packages/node/src/sync/stream/NodeRemote.ts @@ -13,7 +13,7 @@ import { } from '@powersync/common'; import { BSON } from 'bson'; import Agent from 'proxy-agent'; -import { ProxyAgent, EnvHttpProxyAgent } from 'undici'; +import { EnvHttpProxyAgent, Dispatcher } from 'undici'; import { WebSocket } from 'ws'; export const STREAMING_POST_TIMEOUT_MS = 30_000; @@ -24,26 +24,26 @@ class NodeFetchProvider extends FetchImplementationProvider { } } -export class NodeRemote extends AbstractRemote { - protected agent: ProxyAgent | undefined; +export type NodeRemoteOptions = AbstractRemoteOptions & { + dispatcher?: Dispatcher; +}; +export class NodeRemote extends AbstractRemote { constructor( protected connector: RemoteConnector, protected logger: ILogger = DEFAULT_REMOTE_LOGGER, - options?: Partial + options?: Partial ) { - // Automatically uses relevant env vars for HTTP - const agent = new EnvHttpProxyAgent(); + // EnvHttpProxyAgent automatically uses relevant env vars for HTTP + const dispatcher = options?.dispatcher ?? new EnvHttpProxyAgent(); super(connector, logger, { ...(options ?? {}), fetchImplementation: options?.fetchImplementation ?? new NodeFetchProvider(), fetchOptions: { - dispatcher: agent + dispatcher } }); - - this.agent = agent; } protected createSocket(url: string): globalThis.WebSocket { From 88eaff3e4b6c0ba152cf21428ac1eb1d7d70cb07 Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Wed, 23 Apr 2025 09:46:59 +0200 Subject: [PATCH 6/7] Merged changesets for common. --- .changeset/thick-lies-invent.md | 1 + .changeset/thirty-humans-shop.md | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 .changeset/thirty-humans-shop.md diff --git a/.changeset/thick-lies-invent.md b/.changeset/thick-lies-invent.md index f3480ddcf..673050cdd 100644 --- a/.changeset/thick-lies-invent.md +++ b/.changeset/thick-lies-invent.md @@ -3,3 +3,4 @@ --- Added `fetchOptions` to AbstractRemoteOptions. Allows consumers to include fields such as `dispatcher` (e.g. for proxy support) to the fetch invocations. +Also ensuring all options provided to `connect()` are passed onwards, allows packages to have their own option definitions for `connect()` and the abstract `generateSyncStreamImplementation()`. diff --git a/.changeset/thirty-humans-shop.md b/.changeset/thirty-humans-shop.md deleted file mode 100644 index 253f68c9e..000000000 --- a/.changeset/thirty-humans-shop.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@powersync/common': patch ---- - -Ensuring all options provided to `connect()` are passed onwards, allows packages to have their own option definitions for `connect()` and the abstract `generateSyncStreamImplementation()`. From b2c0dcd65d3c07f793371c9efe741e2b29e071fc Mon Sep 17 00:00:00 2001 From: Christiaan Landman Date: Wed, 23 Apr 2025 10:38:56 +0200 Subject: [PATCH 7/7] Docs for dispatcher config option. --- packages/node/src/db/PowerSyncDatabase.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/node/src/db/PowerSyncDatabase.ts b/packages/node/src/db/PowerSyncDatabase.ts index 94f577919..c00a44f39 100644 --- a/packages/node/src/db/PowerSyncDatabase.ts +++ b/packages/node/src/db/PowerSyncDatabase.ts @@ -25,6 +25,10 @@ export type NodePowerSyncDatabaseOptions = PowerSyncDatabaseOptions & { }; export type NodeAdditionalConnectionOptions = AdditionalConnectionOptions & { + /** + * Optional custom dispatcher for HTTP connections (e.g. using undici). + * Only used when the connection method is SyncStreamConnectionMethod.HTTP + */ dispatcher?: Dispatcher; };