From 6ed725d65821db9d7f6a1fbfc8c4cd11615f7634 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Nov 2024 17:23:58 +0200 Subject: [PATCH 1/3] Properly replicate additional MongoDB types: Decimal, RegExp, MinKey, MaxKey. --- modules/module-mongodb/package.json | 2 +- .../src/api/MongoRouteAPIAdapter.ts | 8 +++++ .../src/replication/MongoRelation.ts | 12 ++++++++ .../test/src/mongo_test.test.ts | 30 ++++++++++++++----- modules/module-mongodb/vitest.config.ts | 8 ++++- 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index 16b65a4ab..d928ec513 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -13,7 +13,7 @@ "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", "clean": "rm -rf ./dist && tsc -b --clean", - "test": "vitest --no-threads" + "test": "vitest" }, "exports": { ".": { diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index dd611ce9b..b9b740e24 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -313,6 +313,14 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { return 'Binary'; } else if (data instanceof mongo.Long) { return 'Long'; + } else if (data instanceof RegExp) { + return 'RegExp'; + } else if (data instanceof mongo.MinKey) { + return 'MinKey'; + } else if (data instanceof mongo.MaxKey) { + return 'MaxKey'; + } else if (data instanceof mongo.Decimal128) { + return 'Decimal'; } else if (Array.isArray(data)) { return 'Array'; } else if (data instanceof Uint8Array) { diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index 267674895..1a418a6ae 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -62,6 +62,12 @@ export function toMongoSyncRulesValue(data: any): SqliteValue { return new Uint8Array(data.buffer); } else if (data instanceof mongo.Long) { return data.toBigInt(); + } else if (data instanceof mongo.Decimal128) { + return data.toString(); + } else if (data instanceof mongo.MinKey || data instanceof mongo.MaxKey) { + return null; + } else if (data instanceof RegExp) { + return JSON.stringify({ pattern: data.source, options: data.flags }); } else if (Array.isArray(data)) { // We may be able to avoid some parse + stringify cycles here for JsonSqliteContainer. return JSONBig.stringify(data.map((element) => filterJsonData(element))); @@ -112,6 +118,12 @@ function filterJsonData(data: any, depth = 0): any { return undefined; } else if (data instanceof mongo.Long) { return data.toBigInt(); + } else if (data instanceof mongo.Decimal128) { + return data.toString(); + } else if (data instanceof mongo.MinKey || data instanceof mongo.MaxKey) { + return data._bsontype; + } else if (data instanceof mongo.BSONRegExp) { + return JSON.stringify({ pattern: data.pattern, options: data.options }); } else if (Array.isArray(data)) { return data.map((element) => filterJsonData(element, depth + 1)); } else if (ArrayBuffer.isView(data)) { diff --git a/modules/module-mongodb/test/src/mongo_test.test.ts b/modules/module-mongodb/test/src/mongo_test.test.ts index ed03ebcd9..c3b2b15c5 100644 --- a/modules/module-mongodb/test/src/mongo_test.test.ts +++ b/modules/module-mongodb/test/src/mongo_test.test.ts @@ -23,14 +23,18 @@ describe('mongo data types', () => { int2: 1000, int4: 1000000, int8: 9007199254740993n, - float: 3.14 + float: 3.14, + decimal: new mongo.Decimal128('3.14') }, { _id: 2 as any, nested: { test: 'thing' } }, { _id: 3 as any, date: new Date('2023-03-06 15:47+02') }, { _id: 4 as any, timestamp: mongo.Timestamp.fromBits(123, 456), - objectId: mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb') + objectId: mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb'), + regexp: new mongo.BSONRegExp('test', 'i'), + minKey: new mongo.MinKey(), + maxKey: new mongo.MaxKey() } ]); } @@ -47,14 +51,18 @@ describe('mongo data types', () => { int2: [1000], int4: [1000000], int8: [9007199254740993n], - float: [3.14] + float: [3.14], + decimal: [new mongo.Decimal128('3.14')] }, { _id: 2 as any, nested: [{ test: 'thing' }] }, { _id: 3 as any, date: [new Date('2023-03-06 15:47+02')] }, { _id: 10 as any, timestamp: [mongo.Timestamp.fromBits(123, 456)], - objectId: [mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')] + objectId: [mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')], + regexp: [new mongo.BSONRegExp('test', 'i')], + minKey: [new mongo.MinKey()], + maxKey: [new mongo.MaxKey()] } ]); } @@ -70,7 +78,8 @@ describe('mongo data types', () => { int4: 1000000n, int8: 9007199254740993n, float: 3.14, - null: null + null: null, + decimal: '3.14' }); expect(transformed[1]).toMatchObject({ _id: 2n, @@ -85,7 +94,10 @@ describe('mongo data types', () => { expect(transformed[3]).toMatchObject({ _id: 4n, objectId: '66e834cc91d805df11fa0ecb', - timestamp: 1958505087099n + timestamp: 1958505087099n, + regexp: '{"pattern":"test","options":"i"}', + minKey: null, + maxKey: null }); } @@ -220,7 +232,7 @@ describe('mongo data types', () => { const schema = await adapter.getConnectionSchema(); const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0]; expect(dbSchema).not.toBeNull(); - expect(dbSchema.tables).toEqual([ + expect(dbSchema.tables).toMatchObject([ { name: 'test_data', columns: [ @@ -228,13 +240,17 @@ describe('mongo data types', () => { { name: 'bool', sqlite_type: 4, internal_type: 'Boolean' }, { name: 'bytea', sqlite_type: 1, internal_type: 'Binary' }, { name: 'date', sqlite_type: 2, internal_type: 'Date' }, + { name: 'decimal', sqlite_type: 2, internal_type: 'Decimal' }, { name: 'float', sqlite_type: 8, internal_type: 'Double' }, { name: 'int2', sqlite_type: 4, internal_type: 'Integer' }, { name: 'int4', sqlite_type: 4, internal_type: 'Integer' }, { name: 'int8', sqlite_type: 4, internal_type: 'Long' }, + { name: 'maxKey', sqlite_type: 0, internal_type: 'MaxKey' }, + { name: 'minKey', sqlite_type: 0, internal_type: 'MinKey' }, { name: 'nested', sqlite_type: 2, internal_type: 'Object' }, { name: 'null', sqlite_type: 0, internal_type: 'Null' }, { name: 'objectId', sqlite_type: 2, internal_type: 'ObjectId' }, + { name: 'regexp', sqlite_type: 2, internal_type: 'RegExp' }, { name: 'text', sqlite_type: 2, internal_type: 'String' }, { name: 'timestamp', sqlite_type: 4, internal_type: 'Timestamp' }, { name: 'uuid', sqlite_type: 2, internal_type: 'UUID' } diff --git a/modules/module-mongodb/vitest.config.ts b/modules/module-mongodb/vitest.config.ts index b392696b7..7a39c1f71 100644 --- a/modules/module-mongodb/vitest.config.ts +++ b/modules/module-mongodb/vitest.config.ts @@ -4,6 +4,12 @@ import tsconfigPaths from 'vite-tsconfig-paths'; export default defineConfig({ plugins: [tsconfigPaths()], test: { - setupFiles: './test/src/setup.ts' + setupFiles: './test/src/setup.ts', + poolOptions: { + threads: { + singleThread: true + } + }, + pool: 'threads' } }); From 14bdceba654b60a62268a918a508762a4641c577 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Nov 2024 17:26:38 +0200 Subject: [PATCH 2/3] Test mongodb on GA. --- .github/workflows/test.yml | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3705cd600..5d904012e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -143,3 +143,56 @@ jobs: - name: Test run: pnpm test --filter='./modules/module-postgres' + + run-mongodb-tests: + name: MongoDB Test + runs-on: ubuntu-latest + needs: run-core-tests + + strategy: + fail-fast: false + matrix: + mongodb-version: ['6.0', '7.0'] + + steps: + - uses: actions/checkout@v4 + + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.8.0 + with: + mongodb-version: ${{ matrix.mongodb-version }} + mongodb-replica-set: test-rs + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + + - uses: pnpm/action-setup@v4 + name: Install pnpm + with: + version: 9 + run_install: false + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - uses: actions/cache@v3 + name: Setup pnpm cache + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + run: pnpm install + + - name: Build + shell: bash + run: pnpm build + + - name: Test + run: pnpm test --filter='./modules/module-mongodb' From 3bca30f35aeacfcde164d65f6e9092e7c7af87c0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Nov 2024 17:42:43 +0200 Subject: [PATCH 3/3] Also test with MongoDB 8.0. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fedcf60e5..5ef3c3f1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -217,7 +217,7 @@ jobs: strategy: fail-fast: false matrix: - mongodb-version: ['6.0', '7.0'] + mongodb-version: ['6.0', '7.0', '8.0'] steps: - uses: actions/checkout@v4