Skip to content

Commit a189496

Browse files
simolus3simolus3
committed
Fix encoding
1 parent c53f99d commit a189496

File tree

7 files changed

+158
-90
lines changed

7 files changed

+158
-90
lines changed

‎modules/module-postgres/src/types/custom.ts‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,20 @@ export class PostgresTypeCache {
1313
let pending = oids.filter((id) => !(id in Object.values(pgwire.PgTypeOid)));
1414
// For details on columns, see https://www.postgresql.org/docs/current/catalog-pg-type.html
1515
const statement = `
16-
SELECT oid, pg_type.typtype,
17-
CASE pg_type.typtype
18-
WHEN 'd' THEN json_build_object('type', pg_type.typbasetype)
16+
SELECT oid, t.typtype,
17+
CASE t.typtype
18+
WHEN 'b' THEN json_build_object('element_type', t.typelem, 'delim', (SELECT typdelim FROM pg_type i WHERE i.oid = t.typelem))
19+
WHEN 'd' THEN json_build_object('type', t.typbasetype)
1920
WHEN 'c' THEN json_build_object(
2021
'elements',
2122
(SELECT json_agg(json_build_object('name', a.attname, 'type', a.atttypid))
2223
FROM pg_attribute a
23-
WHERE a.attrelid = pg_type.typrelid)
24+
WHERE a.attrelid = t.typrelid)
2425
)
2526
ELSE NULL
2627
END AS desc
27-
FROM pg_type
28-
WHERE pg_type.oid = ANY($1)
28+
FROM pg_type t
29+
WHERE t.oid = ANY($1)
2930
`;
3031

3132
while (pending.length != 0) {
@@ -43,12 +44,18 @@ WHERE pg_type.oid = ANY($1)
4344
const desc = JSON.parse(row.desc);
4445

4546
switch (row.typtype) {
46-
case 'd':
47-
// For domain values like CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5), we sync
48-
// the inner type (pg_type.typbasetype).
49-
const inner = Number(desc.type);
50-
this.registry.setDomainType(oid, inner);
51-
requireType(inner);
47+
case 'b':
48+
const { element_type, delim } = desc;
49+
50+
if (!this.registry.knows(oid)) {
51+
// This type is an array of another custom type.
52+
this.registry.set(oid, {
53+
type: 'array',
54+
innerId: Number(element_type),
55+
separatorCharCode: (delim as string).charCodeAt(0),
56+
sqliteType: () => 'text' // Since it's JSON
57+
});
58+
}
5259
break;
5360
case 'c':
5461
// For composite types, we sync the JSON representation.
@@ -65,6 +72,13 @@ WHERE pg_type.oid = ANY($1)
6572
sqliteType: () => 'text' // Since it's JSON
6673
});
6774
break;
75+
case 'd':
76+
// For domain values like CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5), we sync
77+
// the inner type (pg_type.typbasetype).
78+
const inner = Number(desc.type);
79+
this.registry.setDomainType(oid, inner);
80+
requireType(inner);
81+
break;
6882
}
6983
}
7084

‎modules/module-postgres/src/types/registry.ts‎

Lines changed: 5 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ class CustomTypeValue extends CustomSqliteValue {
6464
toSqliteValue(context: CompatibilityContext): SqliteValue {
6565
if (context.isEnabled(CompatibilityOption.customTypes)) {
6666
try {
67-
const value = toSyncRulesValue(this.cache.decodeWithCustomTypes(this.rawValue, this.oid));
67+
const rawValue = this.cache.decodeWithCustomTypes(this.rawValue, this.oid);
68+
const value = toSyncRulesValue(rawValue);
6869
return applyValueContext(value, context);
6970
} catch (_e) {
7071
return this.rawValue;
@@ -211,8 +212,6 @@ export class CustomTypeRegistry {
211212
pendingNestedStructure = null;
212213
},
213214
onValue: (raw) => {
214-
// this isn't an array or a composite (because in that case we'd make maybeParseSubStructure return nested
215-
// delimiters and this wouldn't get called).
216215
pushParsedValue(raw == null ? null : this.decodeWithCustomTypes(raw, resolveCurrentStructureTypeId()));
217216
},
218217
onStructureEnd: () => {
@@ -229,11 +228,6 @@ export class CustomTypeRegistry {
229228
}
230229
},
231230
maybeParseSubStructure: (firstChar: number) => {
232-
if (firstChar != pgwire.CHAR_CODE_LEFT_BRACE && firstChar != pgwire.CHAR_CODE_LEFT_PAREN) {
233-
// Fast path - definitely not a sub-structure.
234-
return null;
235-
}
236-
237231
const top = stateStack[stateStack.length - 1];
238232
if (top.type == 'array' && firstChar == pgwire.CHAR_CODE_LEFT_BRACE) {
239233
// Postgres arrays are natively multidimensional - so if we're in an array, we can always parse sub-arrays
@@ -242,16 +236,7 @@ export class CustomTypeRegistry {
242236
return this.delimitersFor(top);
243237
}
244238

245-
const current = this.lookupType(resolveCurrentStructureTypeId());
246-
const structure = this.resolveStructure(current);
247-
if (structure != null) {
248-
const [nestedType, delimiters] = structure;
249-
if (delimiters.openingCharCode == firstChar) {
250-
pendingNestedStructure = nestedType;
251-
return delimiters;
252-
}
253-
}
254-
239+
// If we're in a compound type, nested compound values or arrays are encoded as strings.
255240
return null;
256241
}
257242
}
@@ -275,17 +260,9 @@ export class CustomTypeRegistry {
275260

276261
private delimitersFor(type: ArrayType | CompositeType): pgwire.Delimiters {
277262
if (type.type == 'array') {
278-
return {
279-
openingCharCode: pgwire.CHAR_CODE_LEFT_BRACE,
280-
closingCharCode: pgwire.CHAR_CODE_RIGHT_BRACE,
281-
delimiterCharCode: type.separatorCharCode
282-
};
263+
return pgwire.arrayDelimiters(type.separatorCharCode);
283264
} else {
284-
return {
285-
openingCharCode: pgwire.CHAR_CODE_LEFT_PAREN,
286-
closingCharCode: pgwire.CHAR_CODE_RIGHT_PAREN,
287-
delimiterCharCode: pgwire.CHAR_CODE_COMMA
288-
};
265+
return pgwire.COMPOSITE_DELIMITERS;
289266
}
290267
}
291268

‎modules/module-postgres/test/src/pg_test.test.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -476,12 +476,14 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
476476
try {
477477
await clearTestDb(db);
478478
await db.query(`CREATE DOMAIN rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5);`);
479-
await db.query(`CREATE TYPE composite AS (foo rating_value, bar TEXT);`);
479+
await db.query(`CREATE TYPE composite AS (foo rating_value[], bar TEXT);`);
480+
await db.query(`CREATE TYPE nested_composite AS (a BOOLEAN, b composite);`);
480481

481482
await db.query(`CREATE TABLE test_custom(
482483
id serial primary key,
483484
rating rating_value,
484-
composite composite
485+
composite composite,
486+
nested_composite nested_composite
485487
);`);
486488

487489
const slotName = 'test_slot';
@@ -498,10 +500,11 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
498500

499501
await db.query(`
500502
INSERT INTO test_custom
501-
(rating, composite)
503+
(rating, composite, nested_composite)
502504
VALUES (
503505
1,
504-
(2, 'bar')
506+
(ARRAY[2,3], 'bar'),
507+
(TRUE, (ARRAY[2,3], 'bar'))
505508
);
506509
`);
507510

@@ -519,12 +522,16 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12'
519522

520523
const oldFormat = applyRowContext(transformed, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY);
521524
expect(oldFormat).toMatchObject({
522-
rating: '1'
525+
rating: '1',
526+
composite: '("{2,3}",bar)',
527+
nested_composite: '(t,"(""{2,3}"",bar)")'
523528
});
524529

525530
const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS));
526531
expect(newFormat).toMatchObject({
527-
rating: 1
532+
rating: 1,
533+
composite: '{"foo":[2.0,3.0],"bar":"bar"}',
534+
nested_composite: '{"a":1,"b":{"foo":[2.0,3.0],"bar":"bar"}}'
528535
});
529536
} finally {
530537
await db.end();

‎modules/module-postgres/test/src/types/registry.test.ts‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ describe('custom type registry', () => {
4646
});
4747

4848
test('structure', () => {
49+
// create type c1 AS (a bool, b integer, c text[]);
4950
registry.set(1337, {
5051
type: 'composite',
5152
sqliteType: () => 'text',
@@ -56,10 +57,12 @@ describe('custom type registry', () => {
5657
]
5758
});
5859

59-
checkResult('(t,123,{foo,bar})', 1337, '(t,123,{foo,bar})', '{"a":1,"b":123,"c":["foo","bar"]}');
60+
// SELECT (TRUE, 123, ARRAY['foo', 'bar'])::c1;
61+
checkResult('(t,123,"{foo,bar}")', 1337, '(t,123,"{foo,bar}")', '{"a":1,"b":123,"c":["foo","bar"]}');
6062
});
6163

6264
test('array of structure', () => {
65+
// create type c1 AS (a bool, b integer, c text[]);
6366
registry.set(1337, {
6467
type: 'composite',
6568
sqliteType: () => 'text',
@@ -71,11 +74,12 @@ describe('custom type registry', () => {
7174
});
7275
registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
7376

77+
// SELECT ARRAY[(TRUE, 123, ARRAY['foo', 'bar']),(FALSE, NULL, ARRAY[]::text[])]::c1[];
7478
checkResult(
75-
'{(t,123,{foo,bar}),(f,0,{})}',
79+
'{"(t,123,\\"{foo,bar}\\")","(f,,{})"}',
7680
1338,
77-
'{(t,123,{foo,bar}),(f,0,{})}',
78-
'[{"a":1,"b":123,"c":["foo","bar"]},{"a":0,"b":0,"c":[]}]'
81+
'{"(t,123,\\"{foo,bar}\\")","(f,,{})"}',
82+
'[{"a":1,"b":123,"c":["foo","bar"]},{"a":0,"b":null,"c":[]}]'
7983
);
8084
});
8185

@@ -94,6 +98,7 @@ describe('custom type registry', () => {
9498
});
9599

96100
test('structure of another structure', () => {
101+
// CREATE TYPE c2 AS (a BOOLEAN, b INTEGER);
97102
registry.set(1337, {
98103
type: 'composite',
99104
sqliteType: () => 'text',
@@ -103,12 +108,14 @@ describe('custom type registry', () => {
103108
]
104109
});
105110
registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' });
111+
// CREATE TYPE c3 (c c2[]);
106112
registry.set(1339, {
107113
type: 'composite',
108114
sqliteType: () => 'text',
109115
members: [{ name: 'c', typeId: 1338 }]
110116
});
111117

112-
checkResult('({(f,2)})', 1339, '({(f,2)})', '{"c":[{"a":0,"b":2}]}');
118+
// SELECT ROW(ARRAY[(FALSE,2)]::c2[])::c3;
119+
checkResult('("{""(f,2)""}")', 1339, '("{""(f,2)""}")', '{"c":[{"a":0,"b":2}]}');
113120
});
114121
});

‎packages/jpgwire/src/pgwire_types.ts‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { JsonContainer } from '@powersync/service-jsonbig';
44
import { TimeValue, type DatabaseInputValue } from '@powersync/service-sync-rules';
55
import { dateToSqlite, lsnMakeComparable, timestampToSqlite, timestamptzToSqlite } from './util.js';
66
import {
7+
arrayDelimiters,
78
CHAR_CODE_COMMA,
89
CHAR_CODE_LEFT_BRACE,
910
CHAR_CODE_RIGHT_BRACE,
@@ -166,11 +167,7 @@ export class PgType {
166167

167168
let results: DatabaseInputValue[];
168169
const stack: DatabaseInputValue[][] = [];
169-
const delimiters: Delimiters = {
170-
openingCharCode: CHAR_CODE_LEFT_BRACE,
171-
closingCharCode: CHAR_CODE_RIGHT_BRACE,
172-
delimiterCharCode: CHAR_CODE_COMMA
173-
};
170+
const delimiters = arrayDelimiters();
174171

175172
const listener: SequenceListener = {
176173
maybeParseSubStructure: function (firstChar: number): Delimiters | null {

‎packages/jpgwire/src/sequence_tokenizer.ts‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ export interface Delimiters {
3131
openingCharCode: number;
3232
closingCharCode: number;
3333
delimiterCharCode: number;
34+
allowEscapingWithDoubleDoubleQuote: boolean;
35+
allowEmpty: boolean;
36+
nullLiteral: string;
3437
}
3538

3639
export interface DecodeSequenceOptions {
@@ -83,6 +86,7 @@ export function decodeSequence(options: DecodeSequenceOptions) {
8386
}
8487

8588
function quotedString(): string {
89+
const start = i;
8690
const charCodes: number[] = [];
8791
let previousWasBackslash = false;
8892

@@ -95,6 +99,15 @@ export function decodeSequence(options: DecodeSequenceOptions) {
9599
charCodes.push(next);
96100
previousWasBackslash = false;
97101
} else if (next == CHAR_CODE_DOUBLE_QUOTE) {
102+
if (i != start && delimiters.allowEscapingWithDoubleDoubleQuote) {
103+
// If the next character is also a double quote, that escapes a single double quote
104+
if (i < source.length - 1 && peek() == CHAR_CODE_DOUBLE_QUOTE) {
105+
i++;
106+
charCodes.push(CHAR_CODE_DOUBLE_QUOTE);
107+
continue;
108+
}
109+
}
110+
98111
break; // End of string.
99112
} else if (next == CHAR_CODE_BACKSLASH) {
100113
previousWasBackslash = true;
@@ -148,12 +161,25 @@ export function decodeSequence(options: DecodeSequenceOptions) {
148161
if (charCode == CHAR_CODE_DOUBLE_QUOTE) {
149162
const value = quotedString();
150163
listener.onValue(value);
164+
} else if (charCode == delimiters.delimiterCharCode || charCode == delimiters.closingCharCode) {
165+
if (!delimiters.allowEmpty) {
166+
error('invalid empty element');
167+
}
168+
169+
listener.onValue('' == delimiters.nullLiteral ? null : '');
170+
if (charCode == delimiters.delimiterCharCode) {
171+
// Since this is a comma, there'll be an element afterwards
172+
currentState = SequenceDecoderState.BEFORE_ELEMENT;
173+
} else {
174+
endStructure();
175+
}
176+
break;
151177
} else {
152178
const behavior = listener.maybeParseSubStructure(charCode);
153179
if (behavior == null) {
154180
// Parse the current cell as one value
155181
const value = unquotedString();
156-
listener.onValue(value == 'NULL' ? null : value);
182+
listener.onValue(value == delimiters.nullLiteral ? null : value);
157183
} else {
158184
currentState = SequenceDecoderState.AFTER_ELEMENT;
159185
listener.onStructureStart();
@@ -198,6 +224,28 @@ export const CHAR_CODE_RIGHT_BRACE = 0x7d;
198224
export const CHAR_CODE_LEFT_PAREN = 0x28;
199225
export const CHAR_CODE_RIGHT_PAREN = 0x29;
200226

227+
// https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO
228+
export function arrayDelimiters(delimiterCharCode: number = CHAR_CODE_COMMA): Delimiters {
229+
return {
230+
openingCharCode: CHAR_CODE_LEFT_BRACE,
231+
closingCharCode: CHAR_CODE_RIGHT_BRACE,
232+
allowEscapingWithDoubleDoubleQuote: false,
233+
nullLiteral: 'NULL',
234+
allowEmpty: false, // Empty values must be escaped
235+
delimiterCharCode
236+
};
237+
}
238+
239+
// https://www.postgresql.org/docs/current/rowtypes.html#ROWTYPES-IO-SYNTAX
240+
export const COMPOSITE_DELIMITERS = Object.freeze({
241+
openingCharCode: CHAR_CODE_LEFT_PAREN,
242+
closingCharCode: CHAR_CODE_RIGHT_PAREN,
243+
delimiterCharCode: CHAR_CODE_COMMA,
244+
allowEscapingWithDoubleDoubleQuote: true,
245+
allowEmpty: true, // Empty values encode NULL
246+
nullLiteral: ''
247+
} satisfies Delimiters);
248+
201249
enum SequenceDecoderState {
202250
BEFORE_SEQUENCE = 1,
203251
BEFORE_ELEMENT_OR_END = 2,

0 commit comments

Comments
 (0)