Skip to content

Commit b9c45e5

Browse files
simolus3simolus3
committed
Also resolve custom types in nested structures
1 parent c521c8d commit b9c45e5

File tree

9 files changed

+127
-85
lines changed

9 files changed

+127
-85
lines changed

‎modules/module-mongodb/src/replication/MongoRelation.ts‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ import { mongo } from '@powersync/lib-service-mongodb';
22
import { storage } from '@powersync/service-core';
33
import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
44
import {
5+
CompatibilityContext,
6+
CustomArray,
7+
CustomObject,
58
CustomSqliteValue,
69
DatabaseInputValue,
710
SqliteInputRow,
@@ -69,7 +72,7 @@ export function toMongoSyncRulesValue(data: any): SqliteInputValue {
6972
return data.toHexString();
7073
} else if (data instanceof Date) {
7174
const isoString = data.toISOString();
72-
return new TimeValue(isoString.replace('T', ' '), isoString);
75+
return new TimeValue(isoString);
7376
} else if (data instanceof mongo.Binary) {
7477
return new Uint8Array(data.buffer);
7578
} else if (data instanceof mongo.Long) {
@@ -81,25 +84,21 @@ export function toMongoSyncRulesValue(data: any): SqliteInputValue {
8184
} else if (data instanceof RegExp) {
8285
return JSON.stringify({ pattern: data.source, options: data.flags });
8386
} else if (Array.isArray(data)) {
84-
return CustomSqliteValue.wrapArray(data.map((element) => filterJsonData(element)));
87+
return new CustomArray(data, filterJsonData);
8588
} else if (data instanceof Uint8Array) {
8689
return data;
8790
} else if (data instanceof JsonContainer) {
8891
return data.toString();
8992
} else if (typeof data == 'object') {
90-
let record: Record<string, any> = {};
91-
for (let key of Object.keys(data)) {
92-
record[key] = filterJsonData(data[key]);
93-
}
94-
return JSONBig.stringify(record);
93+
return new CustomObject(data, filterJsonData);
9594
} else {
9695
return null;
9796
}
9897
}
9998

10099
const DEPTH_LIMIT = 20;
101100

102-
function filterJsonData(data: any, depth = 0): DatabaseInputValue | undefined {
101+
function filterJsonData(data: any, context: CompatibilityContext, depth = 0): any {
103102
const autoBigNum = true;
104103
if (depth > DEPTH_LIMIT) {
105104
// This is primarily to prevent infinite recursion
@@ -126,7 +125,7 @@ function filterJsonData(data: any, depth = 0): DatabaseInputValue | undefined {
126125
return data;
127126
} else if (data instanceof Date) {
128127
const isoString = data.toISOString();
129-
return new TimeValue(isoString.replace('T', ' '), isoString);
128+
return new TimeValue(isoString).toSqliteValue(context);
130129
} else if (data instanceof mongo.ObjectId) {
131130
return data.toHexString();
132131
} else if (data instanceof mongo.UUID) {
@@ -142,16 +141,18 @@ function filterJsonData(data: any, depth = 0): DatabaseInputValue | undefined {
142141
} else if (data instanceof RegExp) {
143142
return { pattern: data.source, options: data.flags };
144143
} else if (Array.isArray(data)) {
145-
return CustomSqliteValue.wrapArray(data.map((element) => filterJsonData(element, depth + 1)));
144+
return data.map((element) => filterJsonData(element, context, depth + 1));
146145
} else if (ArrayBuffer.isView(data)) {
147146
return undefined;
147+
} else if (data instanceof CustomSqliteValue) {
148+
return data.toSqliteValue(context);
148149
} else if (data instanceof JsonContainer) {
149150
// Can be stringified directly when using our JSONBig implementation
150151
return data;
151152
} else if (typeof data == 'object') {
152153
let record: Record<string, any> = {};
153154
for (let key of Object.keys(data)) {
154-
record[key] = filterJsonData(data[key], depth + 1);
155+
record[key] = filterJsonData(data[key], context, depth + 1);
155156
}
156157
return record;
157158
} else {

‎modules/module-mongodb/test/src/mongo_test.test.ts‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { mongo } from '@powersync/lib-service-mongodb';
2-
import { CustomSqliteValue, SqliteInputRow, SqliteRow, SqlSyncRules, TimeValue } from '@powersync/service-sync-rules';
2+
import { applyRowContext, CompatibilityContext, SqliteInputRow, SqlSyncRules } from '@powersync/service-sync-rules';
33
import { describe, expect, test } from 'vitest';
44

55
import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js';
@@ -138,8 +138,10 @@ describe('mongo data types', () => {
138138
]);
139139
}
140140

141-
function checkResults(transformed: Record<string, any>[]) {
142-
expect(transformed[0]).toMatchObject({
141+
function checkResults(transformed: SqliteInputRow[]) {
142+
const sqliteValue = transformed.map((e) => applyRowContext(e, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY));
143+
144+
expect(sqliteValue[0]).toMatchObject({
143145
_id: 1n,
144146
text: 'text',
145147
uuid: 'baeb2514-4c57-436d-b3cc-c1256211656d',
@@ -152,17 +154,17 @@ describe('mongo data types', () => {
152154
null: null,
153155
decimal: '3.14'
154156
});
155-
expect(transformed[1]).toMatchObject({
157+
expect(sqliteValue[1]).toMatchObject({
156158
_id: 2n,
157159
nested: '{"test":"thing"}'
158160
});
159161

160-
expect(transformed[2]).toMatchObject({
162+
expect(sqliteValue[2]).toMatchObject({
161163
_id: 3n,
162-
date: new TimeValue('2023-03-06 13:47:00.000Z', '2023-03-06T13:47:00.000Z')
164+
date: '2023-03-06 13:47:00.000Z'
163165
});
164166

165-
expect(transformed[3]).toMatchObject({
167+
expect(sqliteValue[3]).toMatchObject({
166168
_id: 4n,
167169
objectId: '66e834cc91d805df11fa0ecb',
168170
timestamp: 1958505087099n,
@@ -177,9 +179,9 @@ describe('mongo data types', () => {
177179
});
178180

179181
// This must specifically be null, and not undefined.
180-
expect(transformed[4].undefined).toBeNull();
182+
expect(sqliteValue[4].undefined).toBeNull();
181183

182-
expect(transformed[5]).toMatchObject({
184+
expect(sqliteValue[5]).toMatchObject({
183185
_id: 6n,
184186
int4: -1n,
185187
int8: -9007199254740993n,
@@ -188,8 +190,10 @@ describe('mongo data types', () => {
188190
});
189191
}
190192

191-
function checkResultsNested(transformed: Record<string, any>[]) {
192-
expect(transformed[0]).toMatchObject({
193+
function checkResultsNested(transformed: SqliteInputRow[]) {
194+
const sqliteValue = transformed.map((e) => applyRowContext(e, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY));
195+
196+
expect(sqliteValue[0]).toMatchObject({
193197
_id: 1n,
194198
text: `["text"]`,
195199
uuid: '["baeb2514-4c57-436d-b3cc-c1256211656d"]',
@@ -204,30 +208,30 @@ describe('mongo data types', () => {
204208

205209
// Note: Depending on to what extent we use the original postgres value, the whitespace may change, and order may change.
206210
// We do expect that decimals and big numbers are preserved.
207-
expect(transformed[1]).toMatchObject({
211+
expect(sqliteValue[1]).toMatchObject({
208212
_id: 2n,
209213
nested: '[{"test":"thing"}]'
210214
});
211215

212-
expect(transformed[2]).toMatchObject({
216+
expect(sqliteValue[2]).toMatchObject({
213217
_id: 3n,
214-
date: CustomSqliteValue.wrapArray([new TimeValue('2023-03-06 13:47:00.000Z', '2023-03-06T13:47:00.000Z')])
218+
date: '["2023-03-06 13:47:00.000Z"]'
215219
});
216220

217-
expect(transformed[3]).toMatchObject({
221+
expect(sqliteValue[3]).toMatchObject({
218222
_id: 5n,
219223
undefined: '[null]'
220224
});
221225

222-
expect(transformed[4]).toMatchObject({
226+
expect(sqliteValue[4]).toMatchObject({
223227
_id: 6n,
224228
int4: '[-1]',
225229
int8: '[-9007199254740993]',
226230
float: '[-3.14]',
227231
decimal: '["-3.14"]'
228232
});
229233

230-
expect(transformed[5]).toMatchObject({
234+
expect(sqliteValue[5]).toMatchObject({
231235
_id: 10n,
232236
objectId: '["66e834cc91d805df11fa0ecb"]',
233237
timestamp: '[1958505087099]',

‎modules/module-postgres/test/src/pg_test.test.ts‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { constructAfterRecord } from '@module/utils/pgwire_utils.js';
22
import * as pgwire from '@powersync/service-jpgwire';
3-
import { CustomSqliteValue, SqliteInputRow, SqliteRow, TimeValue } from '@powersync/service-sync-rules';
3+
import { applyRowContext, CompatibilityContext, SqliteInputRow, TimeValue } from '@powersync/service-sync-rules';
44
import { describe, expect, test } from 'vitest';
55
import { clearTestDb, connectPgPool, connectPgWire, TEST_URI } from './util.js';
66
import { WalStream } from '@module/replication/WalStream.js';
@@ -159,8 +159,8 @@ VALUES(10, ARRAY['null']::TEXT[]);
159159
id: 3n,
160160
date: '2023-03-06',
161161
time: '15:47:00',
162-
timestamp: new TimeValue('2023-03-06 15:47:00', '2023-03-06T15:47:00'),
163-
timestamptz: new TimeValue('2023-03-06 13:47:00Z', '2023-03-06T13:47:00Z')
162+
timestamp: new TimeValue('2023-03-06T15:47:00'),
163+
timestamptz: new TimeValue('2023-03-06T13:47:00Z')
164164
});
165165

166166
expect(transformed[3]).toMatchObject({
@@ -176,25 +176,25 @@ VALUES(10, ARRAY['null']::TEXT[]);
176176
id: 5n,
177177
date: '0000-01-01',
178178
time: '00:00:00',
179-
timestamp: new TimeValue('0000-01-01 00:00:00', '0000-01-01T00:00:00'),
180-
timestamptz: new TimeValue('0000-01-01 00:00:00Z', '0000-01-01T00:00:00Z')
179+
timestamp: new TimeValue('0000-01-01T00:00:00'),
180+
timestamptz: new TimeValue('0000-01-01T00:00:00Z')
181181
});
182182

183183
expect(transformed[5]).toMatchObject({
184184
id: 6n,
185-
timestamp: new TimeValue('1970-01-01 00:00:00', '1970-01-01T00:00:00'),
186-
timestamptz: new TimeValue('1970-01-01 00:00:00Z', '1970-01-01T00:00:00Z')
185+
timestamp: new TimeValue('1970-01-01T00:00:00'),
186+
timestamptz: new TimeValue('1970-01-01T00:00:00Z')
187187
});
188188

189189
expect(transformed[6]).toMatchObject({
190190
id: 7n,
191-
timestamp: new TimeValue('9999-12-31 23:59:59', '9999-12-31T23:59:59'),
192-
timestamptz: new TimeValue('9999-12-31 23:59:59Z', '9999-12-31T23:59:59Z')
191+
timestamp: new TimeValue('9999-12-31T23:59:59'),
192+
timestamptz: new TimeValue('9999-12-31T23:59:59Z')
193193
});
194194

195195
expect(transformed[7]).toMatchObject({
196196
id: 8n,
197-
timestamptz: new TimeValue('0022-02-03 09:13:14Z', '0022-02-03T09:13:14Z')
197+
timestamptz: new TimeValue('0022-02-03T09:13:14Z')
198198
});
199199

200200
expect(transformed[8]).toMatchObject({
@@ -235,11 +235,8 @@ VALUES(10, ARRAY['null']::TEXT[]);
235235
id: 3n,
236236
date: `["2023-03-06"]`,
237237
time: `["15:47:00"]`,
238-
timestamp: CustomSqliteValue.wrapArray([new TimeValue('2023-03-06 15:47:00', '2023-03-06T15:47:00')]),
239-
timestamptz: CustomSqliteValue.wrapArray([
240-
new TimeValue('2023-03-06 13:47:00Z', '2023-03-06T13:47:00Z'),
241-
new TimeValue('2023-03-06 13:47:00.12345Z', '2023-03-06T13:47:00.12345Z')
242-
])
238+
timestamp: '["2023-03-06 15:47:00"]',
239+
timestamptz: '["2023-03-06 13:47:00Z","2023-03-06 13:47:00.12345Z"]'
243240
});
244241

245242
expect(transformed[3]).toMatchObject({
@@ -342,7 +339,7 @@ VALUES(10, ARRAY['null']::TEXT[]);
342339

343340
const transformed = [
344341
...WalStream.getQueryData(pgwire.pgwireRows(await db.query(`SELECT * FROM test_data_arrays ORDER BY id`)))
345-
];
342+
].map((e) => applyRowContext(e, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY));
346343

347344
checkResultArrays(transformed);
348345
} finally {
@@ -418,7 +415,7 @@ VALUES(10, ARRAY['null']::TEXT[]);
418415
const transformed = await getReplicationTx(replicationStream);
419416
await pg.end();
420417

421-
checkResultArrays(transformed);
418+
checkResultArrays(transformed.map((e) => applyRowContext(e, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)));
422419
} finally {
423420
await db.end();
424421
}

‎packages/jpgwire/src/util.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -241,9 +241,9 @@ export function timestamptzToSqlite(source?: string): TimeValue | null {
241241
const match = /^([\d\-]+) ([\d:]+)(\.\d+)?([+-][\d:]+)$/.exec(source);
242242
if (match == null) {
243243
if (source == 'infinity') {
244-
return new TimeValue('9999-12-31 23:59:59Z', '9999-12-31T23:59:59Z');
244+
return new TimeValue('9999-12-31T23:59:59Z');
245245
} else if (source == '-infinity') {
246-
return new TimeValue('0000-01-01 00:00:00Z', '0000-01-01T00:00:00Z');
246+
return new TimeValue('0000-01-01T00:00:00Z');
247247
} else {
248248
return null;
249249
}
@@ -261,7 +261,7 @@ export function timestamptzToSqlite(source?: string): TimeValue | null {
261261
const baseValue = parsed.toISOString().replace('.000', '').replace('Z', '');
262262
const baseText = `${baseValue}${precision ?? ''}Z`;
263263

264-
return new TimeValue(baseText.replace('T', ' '), baseText);
264+
return new TimeValue(baseText);
265265
}
266266

267267
/**
@@ -276,11 +276,11 @@ export function timestampToSqlite(source?: string): TimeValue | null {
276276
return null;
277277
}
278278
if (source == 'infinity') {
279-
return new TimeValue('9999-12-31 23:59:59', '9999-12-31T23:59:59');
279+
return new TimeValue('9999-12-31T23:59:59');
280280
} else if (source == '-infinity') {
281-
return new TimeValue('0000-01-01 00:00:00', '0000-01-01T00:00:00');
281+
return new TimeValue('0000-01-01T00:00:00');
282282
} else {
283-
return new TimeValue(source, source.replace(' ', 'T'));
283+
return new TimeValue(source.replace(' ', 'T'));
284284
}
285285
}
286286
/**

‎packages/sync-rules/src/types/custom_sqlite_value.ts‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,21 +24,30 @@ export abstract class CustomSqliteValue {
2424
abstract toSqliteValue(context: CompatibilityContext): SqliteValue;
2525

2626
abstract get sqliteType(): SqliteValueType;
27+
}
2728

28-
static wrapArray(elements: (DatabaseInputValue | undefined)[]): SqliteInputValue {
29-
const hasCustomValue = elements.some((v) => v instanceof CustomSqliteValue);
30-
if (hasCustomValue) {
31-
// We need access to the compatibility context before encoding contents as JSON.
32-
return new CustomArray(elements);
33-
} else {
34-
// We can encode the array statically.
35-
return JSONBig.stringify(elements);
36-
}
29+
export class CustomArray extends CustomSqliteValue {
30+
constructor(
31+
private readonly elements: any[],
32+
private readonly map: (element: any, context: CompatibilityContext) => void
33+
) {
34+
super();
35+
}
36+
37+
get sqliteType(): SqliteValueType {
38+
return 'text';
39+
}
40+
41+
toSqliteValue(context: CompatibilityContext): SqliteValue {
42+
return JSONBig.stringify(this.elements.map((element) => this.map(element, context)));
3743
}
3844
}
3945

40-
class CustomArray extends CustomSqliteValue {
41-
constructor(private readonly elements: (DatabaseInputValue | undefined)[]) {
46+
export class CustomObject extends CustomSqliteValue {
47+
constructor(
48+
private readonly source: Record<string, any>,
49+
private readonly map: (element: any, context: CompatibilityContext) => void
50+
) {
4251
super();
4352
}
4453

@@ -47,10 +56,10 @@ class CustomArray extends CustomSqliteValue {
4756
}
4857

4958
toSqliteValue(context: CompatibilityContext): SqliteValue {
50-
return JSONBig.stringify(
51-
this.elements.map((element) => {
52-
return element instanceof CustomSqliteValue ? element.toSqliteValue(context) : element;
53-
})
54-
);
59+
let record: Record<string, any> = {};
60+
for (let key of Object.keys(this.source)) {
61+
record[key] = this.map(this.source[key], context);
62+
}
63+
return JSONBig.stringify(record);
5564
}
5665
}

‎packages/sync-rules/src/types/time.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,19 @@ import { CustomSqliteValue } from './custom_sqlite_value.js';
1010
* disabled by default until a major upgrade.
1111
*/
1212
export class TimeValue extends CustomSqliteValue {
13-
// YYYY-MM-DD hh:mm:ss.sss / YYYY-MM-DD hh:mm:ss.sssZ
14-
readonly legacyRepresentation: string;
1513
// YYYY-MM-DDThh:mm:ss.sss / YYYY-MM-DDThh:mm:ss.sssZ
1614
readonly iso8601Representation: string;
1715

18-
constructor(legacyRepresentation: string, iso8601Representation: string) {
16+
constructor(iso8601Representation: string) {
1917
super();
20-
this.legacyRepresentation = legacyRepresentation;
2118
this.iso8601Representation = iso8601Representation;
2219
}
2320

21+
// YYYY-MM-DD hh:mm:ss.sss / YYYY-MM-DD hh:mm:ss.sssZ
22+
public get legacyRepresentation(): string {
23+
return this.iso8601Representation.replace('T', ' ');
24+
}
25+
2426
get sqliteType(): SqliteValueType {
2527
return 'text';
2628
}

0 commit comments

Comments
 (0)