diff --git a/.changeset/swift-trains-behave.md b/.changeset/swift-trains-behave.md new file mode 100644 index 000000000..38901b507 --- /dev/null +++ b/.changeset/swift-trains-behave.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-module-mongodb': minor +'@powersync/lib-service-mongodb': minor +--- + +Shared MongoDB dependency between modules. This should help avoid potential multiple versions of MongoDB being present in a project. diff --git a/libs/lib-mongodb/src/db/mongo.ts b/libs/lib-mongodb/src/db/mongo.ts index 28801408a..f7e84b47c 100644 --- a/libs/lib-mongodb/src/db/mongo.ts +++ b/libs/lib-mongodb/src/db/mongo.ts @@ -77,3 +77,7 @@ export async function waitForAuth(db: mongo.Db) { } } } + +export const isMongoServerError = (error: any): error is mongo.MongoServerError => { + return error instanceof mongo.MongoServerError || error?.name == 'MongoServerError'; +}; diff --git a/libs/lib-mongodb/src/index.ts b/libs/lib-mongodb/src/index.ts index 6e824c538..a5fdd348d 100644 --- a/libs/lib-mongodb/src/index.ts +++ b/libs/lib-mongodb/src/index.ts @@ -6,3 +6,6 @@ export * as locks from './locks/locks-index.js'; export * from './types/types.js'; export * as types from './types/types.js'; + +// Re-export mongodb which can avoid using multiple versions of Mongo in a project +export * as mongo from 'mongodb'; diff --git a/modules/module-mongodb-storage/package.json b/modules/module-mongodb-storage/package.json index 4483a1257..c2b1ff5a9 100644 --- a/modules/module-mongodb-storage/package.json +++ b/modules/module-mongodb-storage/package.json @@ -34,7 +34,6 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/lib-service-mongodb": "workspace:*", - "mongodb": "^6.11.0", "bson": "^6.8.0", "ts-codec": "^1.3.0", "ix": "^5.0.0", diff --git a/modules/module-mongodb-storage/src/migrations/mongo-migration-store.ts b/modules/module-mongodb-storage/src/migrations/mongo-migration-store.ts index 57ea8952a..78d56cf30 100644 --- a/modules/module-mongodb-storage/src/migrations/mongo-migration-store.ts +++ b/modules/module-mongodb-storage/src/migrations/mongo-migration-store.ts @@ -1,12 +1,13 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { migrations } from '@powersync/lib-services-framework'; -import { Db } from 'mongodb'; + import * as path from 'path'; /** * A custom store for node-migrate which is used to save and load migrations that have * been operated on to mongo. */ -export const createMongoMigrationStore = (db: Db): migrations.MigrationStore => { +export const createMongoMigrationStore = (db: mongo.Db): migrations.MigrationStore => { const collection = db.collection('migrations'); return { diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 00ad68f42..9cbf67738 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -1,7 +1,6 @@ import { SqlSyncRules } from '@powersync/service-sync-rules'; import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js'; import { LRUCache } from 'lru-cache/min'; -import * as mongo from 'mongodb'; import * as timers from 'timers/promises'; import { storage, sync, utils } from '@powersync/service-core'; @@ -10,6 +9,8 @@ import { DisposableObserver, logger } from '@powersync/lib-services-framework'; import { v4 as uuid } from 'uuid'; import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; + import { PowerSyncMongo } from './implementation/db.js'; import { SyncRuleDocument } from './implementation/models.js'; import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; @@ -285,7 +286,7 @@ export class MongoBucketStorage async getStorageMetrics(): Promise { const ignoreNotExiting = (e: unknown) => { - if (e instanceof mongo.MongoServerError && e.codeName == 'NamespaceNotFound') { + if (lib_mongo.isMongoServerError(e) && e.codeName == 'NamespaceNotFound') { // Collection doesn't exist - return 0 return [{ storageStats: { size: 0 } }]; } else { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 2686ed229..6e0827a52 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -1,6 +1,6 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { SqlEventDescriptor, SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import * as mongo from 'mongodb'; import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework'; import { SaveOperationTag, storage, utils } from '@powersync/service-core'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 7891804e2..2c28751f1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -1,6 +1,7 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { logger } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; -import { AnyBulkWriteOperation, MaxKey, MinKey } from 'mongodb'; + import { PowerSyncMongo } from './db.js'; import { BucketDataDocument, BucketDataKey } from './models.js'; import { cacheKey } from './OperationBatch.js'; @@ -49,7 +50,7 @@ const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; const DEFAULT_MEMORY_LIMIT_MB = 64; export class MongoCompactor { - private updates: AnyBulkWriteOperation[] = []; + private updates: mongo.AnyBulkWriteOperation[] = []; private idLimitBytes: number; private moveBatchLimit: number; @@ -94,12 +95,12 @@ export class MongoCompactor { let currentState: CurrentBucketState | null = null; - let bucketLower: string | MinKey; - let bucketUpper: string | MaxKey; + let bucketLower: string | mongo.MinKey; + let bucketUpper: string | mongo.MaxKey; if (bucket == null) { - bucketLower = new MinKey(); - bucketUpper = new MaxKey(); + bucketLower = new mongo.MinKey(); + bucketUpper = new mongo.MaxKey(); } else if (bucket.includes('[')) { // Exact bucket name bucketLower = bucket; @@ -114,14 +115,14 @@ export class MongoCompactor { const lowerBound: BucketDataKey = { g: this.group_id, b: bucketLower as string, - o: new MinKey() as any + o: new mongo.MinKey() as any }; // Upper bound is adjusted for each batch let upperBound: BucketDataKey = { g: this.group_id, b: bucketUpper as string, - o: new MaxKey() as any + o: new mongo.MaxKey() as any }; while (true) { @@ -287,7 +288,7 @@ export class MongoCompactor { $gte: { g: this.group_id, b: bucket, - o: new MinKey() as any + o: new mongo.MinKey() as any }, $lte: { g: this.group_id, @@ -349,7 +350,7 @@ export class MongoCompactor { $gte: { g: this.group_id, b: bucket, - o: new MinKey() as any + o: new mongo.MinKey() as any }, $lte: lastOpId! } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts index 0a39e1e3e..b828cf342 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts @@ -1,6 +1,6 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; import { SqlSyncRules } from '@powersync/service-sync-rules'; -import * as mongo from 'mongodb'; import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js'; import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; import { PowerSyncMongo } from './db.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 738ed3fd4..e4268718a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -1,10 +1,9 @@ -import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service-sync-rules'; -import * as bson from 'bson'; -import * as mongo from 'mongodb'; - import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; import { DisposableObserver, logger } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; +import { SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; import * as timers from 'timers/promises'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { PowerSyncMongo } from './db.js'; @@ -498,7 +497,7 @@ export class MongoSyncBucketStorage logger.info(`${this.slot_name} Done clearing data`); return; } catch (e: unknown) { - if (e instanceof mongo.MongoServerError && e.codeName == 'MaxTimeMSExpired') { + if (lib_mongo.isMongoServerError(e) && e.codeName == 'MaxTimeMSExpired') { logger.info( `${this.slot_name} Cleared batch of data in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` ); diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index 7b5c3082e..9ec06afd3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -1,7 +1,7 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { JSONBig } from '@powersync/service-jsonbig'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import * as mongo from 'mongodb'; import { logger } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 5e20d1b89..a32df2ae1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -1,6 +1,6 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; -import * as mongo from 'mongodb'; import { MongoStorageConfig } from '../../types/types.js'; import { diff --git a/modules/module-mongodb-storage/src/storage/implementation/util.ts b/modules/module-mongodb-storage/src/storage/implementation/util.ts index e475217be..0d183a8a6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/util.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/util.ts @@ -1,8 +1,10 @@ -import { storage, utils } from '@powersync/service-core'; import * as bson from 'bson'; import * as crypto from 'crypto'; -import * as mongo from 'mongodb'; import * as uuid from 'uuid'; + +import { mongo } from '@powersync/lib-service-mongodb'; +import { storage, utils } from '@powersync/service-core'; + import { PowerSyncMongo } from './db.js'; import { BucketDataDocument } from './models.js'; diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index 4b49caad9..bb14a6b2f 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -34,7 +34,6 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/lib-service-mongodb": "workspace:*", - "mongodb": "^6.11.0", "bson": "^6.8.0", "ts-codec": "^1.3.0", "uuid": "^9.0.1" diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 65ce0836d..694fb021d 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -1,8 +1,9 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; import { api, ParseSyncRulesOptions, SourceTable } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; -import * as mongo from 'mongodb'; + import { MongoManager } from '../replication/MongoManager.js'; import { constructAfterRecord, createCheckpoint } from '../replication/MongoRelation.js'; import { CHECKPOINTS_COLLECTION } from '../replication/replication-utils.js'; @@ -225,7 +226,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { try { collections = await this.client.db(db.name).listCollections().toArray(); } catch (e) { - if (e instanceof mongo.MongoServerError && e.codeName == 'Unauthorized') { + if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') { // Ignore databases we're not authorized to query return null; } @@ -267,7 +268,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { }); } } catch (e) { - if (e instanceof mongo.MongoServerError && e.codeName == 'Unauthorized') { + if (lib_mongo.isMongoServerError(e) && e.codeName == 'Unauthorized') { // Ignore collections we're not authorized to query continue; } diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 730f39813..a74fd695d 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -1,7 +1,9 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { container, logger } from '@powersync/lib-services-framework'; import { Metrics, SaveOperationTag, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core'; import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; -import * as mongo from 'mongodb'; +import { PostImagesOption } from '../types/types.js'; +import { escapeRegExp } from '../utils.js'; import { MongoManager } from './MongoManager.js'; import { constructAfterRecord, @@ -10,9 +12,7 @@ import { getMongoRelation, mongoLsnToTimestamp } from './MongoRelation.js'; -import { escapeRegExp } from '../utils.js'; import { CHECKPOINTS_COLLECTION } from './replication-utils.js'; -import { PostImagesOption } from '../types/types.js'; export const ZERO_LSN = '0000000000000000'; diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts index 78842fd36..cc6f8e11f 100644 --- a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts @@ -1,10 +1,9 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { container } from '@powersync/lib-services-framework'; -import { ChangeStreamInvalidatedError, ChangeStream } from './ChangeStream.js'; - import { replication } from '@powersync/service-core'; -import { ConnectionManagerFactory } from './ConnectionManagerFactory.js'; -import * as mongo from 'mongodb'; +import { ChangeStream, ChangeStreamInvalidatedError } from './ChangeStream.js'; +import { ConnectionManagerFactory } from './ConnectionManagerFactory.js'; export interface ChangeStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions { connectionFactory: ConnectionManagerFactory; diff --git a/modules/module-mongodb/src/replication/MongoManager.ts b/modules/module-mongodb/src/replication/MongoManager.ts index f005d696e..149071f7f 100644 --- a/modules/module-mongodb/src/replication/MongoManager.ts +++ b/modules/module-mongodb/src/replication/MongoManager.ts @@ -1,4 +1,5 @@ -import * as mongo from 'mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; + import { NormalizedMongoConnectionConfig } from '../types/types.js'; export class MongoManager { diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index 598dee253..02deebd72 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -1,7 +1,8 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; -import { SqliteRow, SqliteValue, toSyncRulesRow } from '@powersync/service-sync-rules'; -import * as mongo from 'mongodb'; import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; +import { SqliteRow, SqliteValue } from '@powersync/service-sync-rules'; + import { CHECKPOINTS_COLLECTION } from './replication-utils.js'; export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor { diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index a35fcd67e..8001dd1fc 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -1,11 +1,12 @@ -import { test_utils } from '@powersync/service-core-tests'; - -import { PostImagesOption } from '@module/types/types.js'; -import { storage } from '@powersync/service-core'; import * as crypto from 'crypto'; -import * as mongo from 'mongodb'; import { setTimeout } from 'node:timers/promises'; import { describe, expect, test, vi } from 'vitest'; + +import { mongo } from '@powersync/lib-service-mongodb'; +import { storage } from '@powersync/service-core'; +import { test_utils } from '@powersync/service-core-tests'; + +import { PostImagesOption } from '@module/types/types.js'; import { ChangeStreamTestContext } from './change_stream_utils.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 2209f4112..7390c5a1c 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -1,11 +1,12 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import { ActiveCheckpoint, BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core'; +import { test_utils } from '@powersync/service-core-tests'; import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js'; import { MongoManager } from '@module/replication/MongoManager.js'; import { createCheckpoint } from '@module/replication/MongoRelation.js'; import { NormalizedMongoConnectionConfig } from '@module/types/types.js'; -import { test_utils } from '@powersync/service-core-tests'; -import * as mongo from 'mongodb'; + import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js'; export class ChangeStreamTestContext { diff --git a/modules/module-mongodb/test/src/mongo_test.test.ts b/modules/module-mongodb/test/src/mongo_test.test.ts index 22d36d493..d458da7b8 100644 --- a/modules/module-mongodb/test/src/mongo_test.test.ts +++ b/modules/module-mongodb/test/src/mongo_test.test.ts @@ -1,11 +1,12 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules'; +import { describe, expect, test } from 'vitest'; + import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js'; import { ChangeStream } from '@module/replication/ChangeStream.js'; import { constructAfterRecord } from '@module/replication/MongoRelation.js'; -import { SqliteRow, SqlSyncRules } from '@powersync/service-sync-rules'; -import * as mongo from 'mongodb'; -import { describe, expect, test } from 'vitest'; -import { clearTestDb, connectMongoData, TEST_CONNECTION_OPTIONS } from './util.js'; import { PostImagesOption } from '@module/types/types.js'; +import { clearTestDb, connectMongoData, TEST_CONNECTION_OPTIONS } from './util.js'; describe('mongo data types', () => { async function setupTable(db: mongo.Db) { diff --git a/modules/module-mongodb/test/src/slow_tests.test.ts b/modules/module-mongodb/test/src/slow_tests.test.ts index 4225cfbba..429561119 100644 --- a/modules/module-mongodb/test/src/slow_tests.test.ts +++ b/modules/module-mongodb/test/src/slow_tests.test.ts @@ -1,7 +1,9 @@ -import { storage } from '@powersync/service-core'; -import * as mongo from 'mongodb'; import { setTimeout } from 'node:timers/promises'; import { describe, expect, test } from 'vitest'; + +import { mongo } from '@powersync/lib-service-mongodb'; +import { storage } from '@powersync/service-core'; + import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; diff --git a/modules/module-mongodb/test/src/util.ts b/modules/module-mongodb/test/src/util.ts index 01312adb5..95cb9001d 100644 --- a/modules/module-mongodb/test/src/util.ts +++ b/modules/module-mongodb/test/src/util.ts @@ -1,7 +1,7 @@ -import * as types from '@module/types/types.js'; - +import { mongo } from '@powersync/lib-service-mongodb'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; -import * as mongo from 'mongodb'; + +import * as types from '@module/types/types.js'; import { env } from './env.js'; export const TEST_URI = env.MONGO_TEST_DATA_URL; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f62b68b26..37d5d994b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -141,9 +141,6 @@ importers: bson: specifier: ^6.8.0 version: 6.10.1 - mongodb: - specifier: ^6.11.0 - version: 6.11.0(socks@2.8.3) ts-codec: specifier: ^1.3.0 version: 1.3.0 @@ -190,9 +187,6 @@ importers: lru-cache: specifier: ^10.2.2 version: 10.4.3 - mongodb: - specifier: ^6.11.0 - version: 6.11.0(socks@2.8.3) ts-codec: specifier: ^1.3.0 version: 1.3.0