Skip to content

Commit 5160778

Browse files
rkistnerrkistner
committed
Merge remote-tracking branch 'origin/feat/modular-replication-architecture' into mongodb-postimages
2 parents 0206aaa + c879923 commit 5160778

File tree

17 files changed

+152
-99
lines changed

17 files changed

+152
-99
lines changed

‎.changeset/gentle-icons-try.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@powersync/service-module-mysql': patch
3+
---
4+
5+
Fixed MySQL version checking to better handle non-semantic version strings

‎.changeset/healthy-rules-arrive.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@powersync/service-module-mysql': patch
3+
---
4+
5+
Fixed mysql schema json parsing

‎.github/workflows/test.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ jobs:
152152
strategy:
153153
fail-fast: false
154154
matrix:
155-
mysql-version: [8.0, 8.4]
155+
mysql-version: [5.7, 8.0, 8.4]
156156

157157
steps:
158158
- uses: actions/checkout@v4
@@ -167,7 +167,8 @@ jobs:
167167
-d mysql:${{ matrix.mysql-version }} \
168168
--log-bin=/var/lib/mysql/mysql-bin.log \
169169
--gtid_mode=ON \
170-
--enforce_gtid_consistency=ON
170+
--enforce_gtid_consistency=ON \
171+
--server-id=1
171172
172173
- name: Start MongoDB
173174
uses: supercharge/[email protected]

‎modules/module-mongodb/src/replication/MongoRelation.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,9 @@ function filterJsonData(data: any, depth = 0): any {
122122
} else if (data instanceof mongo.Decimal128) {
123123
return data.toString();
124124
} else if (data instanceof mongo.MinKey || data instanceof mongo.MaxKey) {
125-
return data._bsontype;
126-
} else if (data instanceof mongo.BSONRegExp) {
127-
return JSON.stringify({ pattern: data.pattern, options: data.options });
125+
return null;
126+
} else if (data instanceof RegExp) {
127+
return { pattern: data.source, options: data.flags };
128128
} else if (Array.isArray(data)) {
129129
return data.map((element) => filterJsonData(element, depth + 1));
130130
} else if (ArrayBuffer.isView(data)) {

‎modules/module-mongodb/test/src/mongo_test.test.ts‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,18 @@ describe('mongo data types', () => {
3434
objectId: mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb'),
3535
regexp: new mongo.BSONRegExp('test', 'i'),
3636
minKey: new mongo.MinKey(),
37-
maxKey: new mongo.MaxKey()
37+
maxKey: new mongo.MaxKey(),
38+
symbol: new mongo.BSONSymbol('test'),
39+
js: new mongo.Code('testcode'),
40+
js2: new mongo.Code('testcode', { foo: 'bar' }),
41+
pointer: new mongo.DBRef('mycollection', mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')),
42+
pointer2: new mongo.DBRef(
43+
'mycollection',
44+
mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb'),
45+
'mydb',
46+
{ foo: 'bar' }
47+
),
48+
undefined: undefined
3849
}
3950
]);
4051
}
@@ -62,7 +73,11 @@ describe('mongo data types', () => {
6273
objectId: [mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')],
6374
regexp: [new mongo.BSONRegExp('test', 'i')],
6475
minKey: [new mongo.MinKey()],
65-
maxKey: [new mongo.MaxKey()]
76+
maxKey: [new mongo.MaxKey()],
77+
symbol: [new mongo.BSONSymbol('test')],
78+
js: [new mongo.Code('testcode')],
79+
pointer: [new mongo.DBRef('mycollection', mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb'))],
80+
undefined: [undefined]
6681
}
6782
]);
6883
}
@@ -97,7 +112,13 @@ describe('mongo data types', () => {
97112
timestamp: 1958505087099n,
98113
regexp: '{"pattern":"test","options":"i"}',
99114
minKey: null,
100-
maxKey: null
115+
maxKey: null,
116+
symbol: 'test',
117+
js: '{"code":"testcode","scope":null}',
118+
js2: '{"code":"testcode","scope":{"foo":"bar"}}',
119+
pointer: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}',
120+
pointer2: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{"foo":"bar"}}',
121+
undefined: null
101122
});
102123
}
103124

@@ -130,7 +151,14 @@ describe('mongo data types', () => {
130151
expect(transformed[3]).toMatchObject({
131152
_id: 10n,
132153
objectId: '["66e834cc91d805df11fa0ecb"]',
133-
timestamp: '[1958505087099]'
154+
timestamp: '[1958505087099]',
155+
regexp: '[{"pattern":"test","options":"i"}]',
156+
symbol: '["test"]',
157+
js: '[{"code":"testcode","scope":null}]',
158+
pointer: '[{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}]',
159+
minKey: '[null]',
160+
maxKey: '[null]',
161+
undefined: '[null]'
134162
});
135163
}
136164

@@ -245,14 +273,23 @@ describe('mongo data types', () => {
245273
{ name: 'int2', sqlite_type: 4, internal_type: 'Integer' },
246274
{ name: 'int4', sqlite_type: 4, internal_type: 'Integer' },
247275
{ name: 'int8', sqlite_type: 4, internal_type: 'Long' },
276+
// We can fix these later
277+
{ name: 'js', sqlite_type: 2, internal_type: 'Object' },
278+
{ name: 'js2', sqlite_type: 2, internal_type: 'Object' },
248279
{ name: 'maxKey', sqlite_type: 0, internal_type: 'MaxKey' },
249280
{ name: 'minKey', sqlite_type: 0, internal_type: 'MinKey' },
250281
{ name: 'nested', sqlite_type: 2, internal_type: 'Object' },
251282
{ name: 'null', sqlite_type: 0, internal_type: 'Null' },
252283
{ name: 'objectId', sqlite_type: 2, internal_type: 'ObjectId' },
284+
// We can fix these later
285+
{ name: 'pointer', sqlite_type: 2, internal_type: 'Object' },
286+
{ name: 'pointer2', sqlite_type: 2, internal_type: 'Object' },
253287
{ name: 'regexp', sqlite_type: 2, internal_type: 'RegExp' },
288+
// Can fix this later
289+
{ name: 'symbol', sqlite_type: 2, internal_type: 'String' },
254290
{ name: 'text', sqlite_type: 2, internal_type: 'String' },
255291
{ name: 'timestamp', sqlite_type: 4, internal_type: 'Timestamp' },
292+
{ name: 'undefined', sqlite_type: 0, internal_type: 'Null' },
256293
{ name: 'uuid', sqlite_type: 2, internal_type: 'UUID' }
257294
]
258295
}

‎modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@ import * as sync_rules from '@powersync/service-sync-rules';
44
import * as service_types from '@powersync/service-types';
55
import mysql from 'mysql2/promise';
66
import * as common from '../common/common-index.js';
7-
import * as mysql_utils from '../utils/mysql_utils.js';
7+
import * as mysql_utils from '../utils/mysql-utils.js';
88
import * as types from '../types/types.js';
99
import { toExpressionTypeFromMySQLType } from '../common/common-index.js';
1010

1111
type SchemaResult = {
1212
schema_name: string;
1313
table_name: string;
14-
columns: Array<{ data_type: string; column_name: string }>;
14+
columns: string;
1515
};
1616

1717
export class MySQLRouteAPIAdapter implements api.RouteAPI {
@@ -327,15 +327,17 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI {
327327
tables: []
328328
});
329329

330+
const columns = JSON.parse(result.columns).map((column: { data_type: string; column_name: string }) => ({
331+
name: column.column_name,
332+
type: column.data_type,
333+
sqlite_type: toExpressionTypeFromMySQLType(column.data_type).typeFlags,
334+
internal_type: column.data_type,
335+
pg_type: column.data_type
336+
}));
337+
330338
schema.tables.push({
331339
name: result.table_name,
332-
columns: result.columns.map((column) => ({
333-
name: column.column_name,
334-
type: column.data_type,
335-
sqlite_type: toExpressionTypeFromMySQLType(column.data_type).typeFlags,
336-
internal_type: column.data_type,
337-
pg_type: column.data_type
338-
}))
340+
columns: columns
339341
});
340342

341343
return hash;

‎modules/module-mysql/src/common/ReplicatedGTID.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import mysql from 'mysql2/promise';
22
import * as uuid from 'uuid';
3-
import * as mysql_utils from '../utils/mysql_utils.js';
3+
import * as mysql_utils from '../utils/mysql-utils.js';
44

55
export type BinLogPosition = {
66
filename: string;

‎modules/module-mysql/src/common/check-source-configuration.ts‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,16 @@
11
import mysqlPromise from 'mysql2/promise';
2-
import * as mysql_utils from '../utils/mysql_utils.js';
2+
import * as mysql_utils from '../utils/mysql-utils.js';
3+
4+
const MIN_SUPPORTED_VERSION = '5.7.0';
35

46
export async function checkSourceConfiguration(connection: mysqlPromise.Connection): Promise<string[]> {
57
const errors: string[] = [];
8+
9+
const version = await mysql_utils.getMySQLVersion(connection);
10+
if (!mysql_utils.isVersionAtLeast(version, MIN_SUPPORTED_VERSION)) {
11+
errors.push(`MySQL versions older than ${MIN_SUPPORTED_VERSION} are not supported. Your version is: ${version}.`);
12+
}
13+
614
const [[result]] = await mysql_utils.retriedQuery({
715
connection,
816
query: `
@@ -48,12 +56,3 @@ export async function checkSourceConfiguration(connection: mysqlPromise.Connecti
4856

4957
return errors;
5058
}
51-
52-
export async function getMySQLVersion(connection: mysqlPromise.Connection): Promise<string> {
53-
const [[versionResult]] = await mysql_utils.retriedQuery({
54-
connection,
55-
query: `SELECT VERSION() as version`
56-
});
57-
58-
return versionResult.version as string;
59-
}

‎modules/module-mysql/src/common/get-replication-columns.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { storage } from '@powersync/service-core';
22
import mysqlPromise from 'mysql2/promise';
3-
import * as mysql_utils from '../utils/mysql_utils.js';
3+
import * as mysql_utils from '../utils/mysql-utils.js';
44

55
export type GetReplicationColumnsOptions = {
66
connection: mysqlPromise.Connection;

‎modules/module-mysql/src/common/read-executed-gtid.ts‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,22 @@
11
import mysqlPromise from 'mysql2/promise';
2-
import * as mysql_utils from '../utils/mysql_utils.js';
3-
import { gte } from 'semver';
4-
2+
import * as mysql_utils from '../utils/mysql-utils.js';
53
import { ReplicatedGTID } from './ReplicatedGTID.js';
6-
import { getMySQLVersion } from './check-source-configuration.js';
74

85
/**
96
* Gets the current master HEAD GTID
107
*/
118
export async function readExecutedGtid(connection: mysqlPromise.Connection): Promise<ReplicatedGTID> {
12-
const version = await getMySQLVersion(connection);
9+
const version = await mysql_utils.getMySQLVersion(connection);
10+
1311
let binlogStatus: mysqlPromise.RowDataPacket;
14-
if (gte(version, '8.4.0')) {
15-
// Get the BinLog status
12+
if (mysql_utils.isVersionAtLeast(version, '8.4.0')) {
13+
// Syntax for the below query changed in 8.4.0
1614
const [[binLogResult]] = await mysql_utils.retriedQuery({
1715
connection,
1816
query: `SHOW BINARY LOG STATUS`
1917
});
2018
binlogStatus = binLogResult;
2119
} else {
22-
// TODO Check if this works for version 5.7
23-
// Get the BinLog status
2420
const [[binLogResult]] = await mysql_utils.retriedQuery({
2521
connection,
2622
query: `SHOW MASTER STATUS`

0 commit comments

Comments
 (0)