diff --git a/.changeset/hip-lamps-draw.md b/.changeset/hip-lamps-draw.md new file mode 100644 index 000000000..2224c6d88 --- /dev/null +++ b/.changeset/hip-lamps-draw.md @@ -0,0 +1,5 @@ +--- +'@powersync/node': minor +--- + +Introduced support for specifying proxy environment variables for the connection methods. For HTTP it supports `HTTP_PROXY` or `HTTPS_PROXY`, and for WebSockets it supports `WS_PROXY` and `WSS_PROXY`. diff --git a/.changeset/thick-lies-invent.md b/.changeset/thick-lies-invent.md new file mode 100644 index 000000000..673050cdd --- /dev/null +++ b/.changeset/thick-lies-invent.md @@ -0,0 +1,6 @@ +--- +'@powersync/common': minor +--- + +Added `fetchOptions` to AbstractRemoteOptions. Allows consumers to include fields such as `dispatcher` (e.g. for proxy support) to the fetch invocations. +Also ensuring all options provided to `connect()` are passed onwards, allows packages to have their own option definitions for `connect()` and the abstract `generateSyncStreamImplementation()`. diff --git a/demos/example-node/src/main.ts b/demos/example-node/src/main.ts index 6dc6e777f..cf6ac280a 100644 --- a/demos/example-node/src/main.ts +++ b/demos/example-node/src/main.ts @@ -1,10 +1,10 @@ -import repl_factory from 'node:repl'; import { once } from 'node:events'; +import repl_factory from 'node:repl'; import { PowerSyncDatabase, SyncStreamConnectionMethod } from '@powersync/node'; import { default as Logger } from 'js-logger'; -import { AppSchema, DemoConnector } from './powersync.js'; import { exit } from 'node:process'; +import { AppSchema, DemoConnector } from './powersync.js'; const main = async () => { const logger = Logger.get('PowerSyncDemo'); diff --git a/packages/common/src/client/AbstractPowerSyncDatabase.ts b/packages/common/src/client/AbstractPowerSyncDatabase.ts index 2e71083e7..036decb3e 100644 --- a/packages/common/src/client/AbstractPowerSyncDatabase.ts +++ b/packages/common/src/client/AbstractPowerSyncDatabase.ts @@ -417,6 +417,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { this.currentStatus = new SyncStatus({ @@ -555,7 +553,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { @@ -633,7 +631,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver { @@ -661,7 +659,7 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver; + protected createSocket(url: string): WebSocket { + return new WebSocket(url); + } + /** * Connects to the sync/stream websocket endpoint */ @@ -249,7 +262,8 @@ export abstract class AbstractRemote { const connector = new RSocketConnector({ transport: new WebsocketClientTransport({ - url: this.options.socketUrlTransformer(request.url) + url: this.options.socketUrlTransformer(request.url), + wsCreator: (url) => this.createSocket(url) }), setup: { keepAlive: KEEP_ALIVE_MS, @@ -421,6 +435,7 @@ export abstract class AbstractRemote { body: JSON.stringify(data), signal: controller.signal, cache: 'no-store', + ...(this.options.fetchOptions ?? {}), ...options.fetchOptions }).catch((ex) => { if (ex.name == 'AbortError') { diff --git a/packages/node/README.md b/packages/node/README.md index c64e2cc78..e06b9d3ff 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -56,6 +56,22 @@ contains everything you need to know to get started implementing PowerSync in yo A simple example using `@powersync/node` is available in the [`demos/example-node/`](../demos/example-node) directory. +# Proxy Support + +This SDK supports HTTP, HTTPS, and WebSocket proxies via environment variables. + +## HTTP Connection Method + +Internally we probe the http environment variables and apply it to fetch requests ([undici](https://www.npmjs.com/package/undici/v/5.6.0)) + +- Set the `HTTPS_PROXY` or `HTTP_PROXY` environment variable to automatically route HTTP requests through a proxy. + +## WEB Socket Connection Method + +Internally the [proxy-agent](https://www.npmjs.com/package/proxy-agent) dependency for WebSocket proxies, which has its own internal code for automatically picking up the appropriate environment variables: + +- Set the `WS_PROXY` or `WSS_PROXY` environment variable to route the webocket connections through a proxy. + # Found a bug or need help? - Join our [Discord server](https://discord.gg/powersync) where you can browse topics from our community, ask questions, share feedback, or just say hello :) diff --git a/packages/node/package.json b/packages/node/package.json index c1eee68fc..223a6a08a 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -51,12 +51,15 @@ "@powersync/common": "workspace:*", "async-lock": "^1.4.0", "bson": "^6.6.0", - "comlink": "^4.4.2" + "comlink": "^4.4.2", + "proxy-agent": "^6.5.0", + "undici": "^7.8.0", + "ws": "^8.18.1" }, "devDependencies": { + "@powersync/drizzle-driver": "workspace:*", "@types/async-lock": "^1.4.0", "drizzle-orm": "^0.35.2", - "@powersync/drizzle-driver": "workspace:*", "rollup": "4.14.3", "typescript": "^5.5.3", "vitest": "^3.0.5" diff --git a/packages/node/src/db/PowerSyncDatabase.ts b/packages/node/src/db/PowerSyncDatabase.ts index 48c5b5fcb..c00a44f39 100644 --- a/packages/node/src/db/PowerSyncDatabase.ts +++ b/packages/node/src/db/PowerSyncDatabase.ts @@ -1,11 +1,14 @@ import { AbstractPowerSyncDatabase, AbstractStreamingSyncImplementation, + AdditionalConnectionOptions, BucketStorageAdapter, DBAdapter, PowerSyncBackendConnector, + PowerSyncConnectionOptions, PowerSyncDatabaseOptions, PowerSyncDatabaseOptionsWithSettings, + RequiredAdditionalConnectionOptions, SqliteBucketStorage, SQLOpenFactory } from '@powersync/common'; @@ -15,11 +18,22 @@ import { NodeStreamingSyncImplementation } from '../sync/stream/NodeStreamingSyn import { BetterSQLite3DBAdapter } from './BetterSQLite3DBAdapter.js'; import { NodeSQLOpenOptions } from './options.js'; +import { Dispatcher } from 'undici'; export type NodePowerSyncDatabaseOptions = PowerSyncDatabaseOptions & { database: DBAdapter | SQLOpenFactory | NodeSQLOpenOptions; }; +export type NodeAdditionalConnectionOptions = AdditionalConnectionOptions & { + /** + * Optional custom dispatcher for HTTP connections (e.g. using undici). + * Only used when the connection method is SyncStreamConnectionMethod.HTTP + */ + dispatcher?: Dispatcher; +}; + +export type NodePowerSyncConnectionOptions = PowerSyncConnectionOptions & NodeAdditionalConnectionOptions; + /** * A PowerSync database which provides SQLite functionality * which is automatically synced. @@ -54,10 +68,18 @@ export class PowerSyncDatabase extends AbstractPowerSyncDatabase { return new SqliteBucketStorage(this.database, AbstractPowerSyncDatabase.transactionMutex); } + connect( + connector: PowerSyncBackendConnector, + options?: PowerSyncConnectionOptions & { dispatcher?: Dispatcher } + ): Promise { + return super.connect(connector, options); + } + protected generateSyncStreamImplementation( - connector: PowerSyncBackendConnector + connector: PowerSyncBackendConnector, + options: NodeAdditionalConnectionOptions ): AbstractStreamingSyncImplementation { - const remote = new NodeRemote(connector); + const remote = new NodeRemote(connector, this.options.logger, { dispatcher: options.dispatcher }); return new NodeStreamingSyncImplementation({ adapter: this.bucketStorageAdapter, diff --git a/packages/node/src/sync/stream/NodeRemote.ts b/packages/node/src/sync/stream/NodeRemote.ts index b0c147484..5e76e0f09 100644 --- a/packages/node/src/sync/stream/NodeRemote.ts +++ b/packages/node/src/sync/stream/NodeRemote.ts @@ -12,6 +12,9 @@ import { RemoteConnector } from '@powersync/common'; import { BSON } from 'bson'; +import Agent from 'proxy-agent'; +import { EnvHttpProxyAgent, Dispatcher } from 'undici'; +import { WebSocket } from 'ws'; export const STREAMING_POST_TIMEOUT_MS = 30_000; @@ -21,18 +24,38 @@ class NodeFetchProvider extends FetchImplementationProvider { } } +export type NodeRemoteOptions = AbstractRemoteOptions & { + dispatcher?: Dispatcher; +}; + export class NodeRemote extends AbstractRemote { constructor( protected connector: RemoteConnector, protected logger: ILogger = DEFAULT_REMOTE_LOGGER, - options?: Partial + options?: Partial ) { + // EnvHttpProxyAgent automatically uses relevant env vars for HTTP + const dispatcher = options?.dispatcher ?? new EnvHttpProxyAgent(); + super(connector, logger, { ...(options ?? {}), - fetchImplementation: options?.fetchImplementation ?? new NodeFetchProvider() + fetchImplementation: options?.fetchImplementation ?? new NodeFetchProvider(), + fetchOptions: { + dispatcher + } }); } + protected createSocket(url: string): globalThis.WebSocket { + return new WebSocket(url, { + // Automatically uses relevant env vars for web sockets + agent: new Agent.ProxyAgent(), + headers: { + 'User-Agent': this.getUserAgent() + } + }) as any as globalThis.WebSocket; // This is compatible in Node environments + } + getUserAgent(): string { return [ super.getUserAgent(), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8a8dd6afd..ea18670dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1807,6 +1807,15 @@ importers: comlink: specifier: ^4.4.2 version: 4.4.2 + proxy-agent: + specifier: ^6.5.0 + version: 6.5.0 + undici: + specifier: ^7.8.0 + version: 7.8.0 + ws: + specifier: ^8.18.1 + version: 8.18.1 devDependencies: '@powersync/drizzle-driver': specifier: workspace:* @@ -9682,10 +9691,6 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} - agent-base@7.1.1: - resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==} - engines: {node: '>= 14'} - agent-base@7.1.3: resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==} engines: {node: '>= 14'} @@ -16142,8 +16147,8 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} - pac-proxy-agent@7.0.2: - resolution: {integrity: sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==} + pac-proxy-agent@7.2.0: + resolution: {integrity: sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==} engines: {node: '>= 14'} pac-resolver@7.0.1: @@ -17076,8 +17081,8 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} - proxy-agent@6.4.0: - resolution: {integrity: sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==} + proxy-agent@6.5.0: + resolution: {integrity: sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==} engines: {node: '>= 14'} proxy-from-env@1.1.0: @@ -18408,8 +18413,8 @@ packages: resolution: {integrity: sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==} engines: {node: '>= 10'} - socks-proxy-agent@8.0.4: - resolution: {integrity: sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==} + socks-proxy-agent@8.0.5: + resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} engines: {node: '>= 14'} socks@2.8.3: @@ -19401,6 +19406,10 @@ packages: resolution: {integrity: sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw==} engines: {node: '>=18.17'} + undici@7.8.0: + resolution: {integrity: sha512-vFv1GA99b7eKO1HG/4RPu2Is3FBTWBrmzqzO0mz+rLxN3yXkE4mqRcb8g8fHxzX4blEysrNZLqg5RbJLqX5buA==} + engines: {node: '>=20.18.1'} + unicode-canonical-property-names-ecmascript@2.0.1: resolution: {integrity: sha512-dA8WbNeb2a6oQzAQ55YlT5vQAWGV9WXOsi3SskE3bcCdM0P4SDd+24zS/OCacdRq5BkdsRj9q3Pg6YyQoxIGqg==} engines: {node: '>=4'} @@ -20384,18 +20393,6 @@ packages: utf-8-validate: optional: true - ws@8.18.0: - resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - ws@8.18.1: resolution: {integrity: sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==} engines: {node: '>=10.0.0'} @@ -26787,11 +26784,11 @@ snapshots: '@npmcli/agent@3.0.0': dependencies: - agent-base: 7.1.1 + agent-base: 7.1.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 10.4.3 - socks-proxy-agent: 8.0.4 + socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color @@ -27328,7 +27325,7 @@ snapshots: debug: 4.4.0(supports-color@8.1.1) extract-zip: 2.0.1 progress: 2.0.3 - proxy-agent: 6.4.0 + proxy-agent: 6.5.0 semver: 7.7.1 tar-fs: 3.0.6 unbzip2-stream: 1.4.3 @@ -28770,7 +28767,9 @@ snapshots: transitivePeerDependencies: - '@babel/core' - '@babel/preset-env' + - bufferutil - supports-color + - utf-8-validate '@react-native/normalize-color@2.1.0': {} @@ -31007,8 +31006,7 @@ snapshots: '@tootallnate/once@2.0.0': {} - '@tootallnate/quickjs-emscripten@0.23.0': - optional: true + '@tootallnate/quickjs-emscripten@0.23.0': {} '@trysound/sax@0.2.0': {} @@ -32316,12 +32314,6 @@ snapshots: transitivePeerDependencies: - supports-color - agent-base@7.1.1: - dependencies: - debug: 4.4.0(supports-color@8.1.1) - transitivePeerDependencies: - - supports-color - agent-base@7.1.3: {} agentkeepalive@4.5.0: @@ -32642,7 +32634,6 @@ snapshots: ast-types@0.13.4: dependencies: tslib: 2.8.1 - optional: true ast-types@0.15.2: dependencies: @@ -33001,8 +32992,7 @@ snapshots: dependencies: safe-buffer: 5.1.2 - basic-ftp@5.0.5: - optional: true + basic-ftp@5.0.5: {} batch@0.6.1: {} @@ -34325,8 +34315,7 @@ snapshots: data-uri-to-buffer@4.0.1: {} - data-uri-to-buffer@6.0.2: - optional: true + data-uri-to-buffer@6.0.2: {} data-urls@3.0.2: dependencies: @@ -34498,7 +34487,6 @@ snapshots: ast-types: 0.13.4 escodegen: 2.1.0 esprima: 4.0.1 - optional: true del-cli@5.1.0: dependencies: @@ -37001,7 +36989,6 @@ snapshots: fs-extra: 11.2.0 transitivePeerDependencies: - supports-color - optional: true getenv@1.0.0: {} @@ -37563,7 +37550,7 @@ snapshots: http-proxy-agent@7.0.2: dependencies: - agent-base: 7.1.1 + agent-base: 7.1.3 debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -37637,7 +37624,7 @@ snapshots: https-proxy-agent@7.0.5: dependencies: - agent-base: 7.1.1 + agent-base: 7.1.3 debug: 4.4.0(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -38941,7 +38928,7 @@ snapshots: whatwg-encoding: 3.1.1 whatwg-mimetype: 4.0.0 whatwg-url: 14.0.0 - ws: 8.18.0 + ws: 8.18.1 xml-name-validator: 5.0.0 transitivePeerDependencies: - bufferutil @@ -41052,8 +41039,7 @@ snapshots: nested-error-stacks@2.0.1: {} - netmask@2.0.2: - optional: true + netmask@2.0.2: {} next@14.2.3(@babel/core@7.26.10)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(sass@1.79.4): dependencies: @@ -41582,7 +41568,7 @@ snapshots: p-try@2.2.0: {} - pac-proxy-agent@7.0.2: + pac-proxy-agent@7.2.0: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.3 @@ -41591,16 +41577,14 @@ snapshots: http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 pac-resolver: 7.0.1 - socks-proxy-agent: 8.0.4 + socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color - optional: true pac-resolver@7.0.1: dependencies: degenerator: 5.0.1 netmask: 2.0.2 - optional: true package-json-from-dist@1.0.1: {} @@ -42644,22 +42628,20 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 - proxy-agent@6.4.0: + proxy-agent@6.5.0: dependencies: agent-base: 7.1.3 debug: 4.4.0(supports-color@8.1.1) http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 7.18.3 - pac-proxy-agent: 7.0.2 + pac-proxy-agent: 7.2.0 proxy-from-env: 1.1.0 - socks-proxy-agent: 8.0.4 + socks-proxy-agent: 8.0.5 transitivePeerDependencies: - supports-color - optional: true - proxy-from-env@1.1.0: - optional: true + proxy-from-env@1.1.0: {} prr@1.0.1: optional: true @@ -44769,9 +44751,9 @@ snapshots: transitivePeerDependencies: - supports-color - socks-proxy-agent@8.0.4: + socks-proxy-agent@8.0.5: dependencies: - agent-base: 7.1.1 + agent-base: 7.1.3 debug: 4.4.0(supports-color@8.1.1) socks: 2.8.3 transitivePeerDependencies: @@ -46041,6 +46023,8 @@ snapshots: undici@6.21.0: {} + undici@7.8.0: {} + unicode-canonical-property-names-ecmascript@2.0.1: {} unicode-emoji-modifier-base@1.0.0: {} @@ -47436,8 +47420,6 @@ snapshots: ws@7.5.10: {} - ws@8.18.0: {} - ws@8.18.1: {} xcode@3.0.1: