From 7eae4edf42798bd853e21bfc4c4b490a9451ee60 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 15:56:48 +0200 Subject: [PATCH 1/6] Clarify type of `OplogEntry.data` --- packages/service-core/src/util/protocol-types.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index d68738cbf..c0a43ef54 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -1,5 +1,5 @@ import * as t from 'ts-codec'; -import { BucketDescription, BucketPriority, SqliteJsonValue } from '@powersync/service-sync-rules'; +import { BucketDescription, BucketPriority } from '@powersync/service-sync-rules'; export const BucketRequest = t.object({ name: t.string, @@ -135,7 +135,7 @@ export interface OplogEntry { op: 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; object_type?: string; object_id?: string; - data?: Record | string | null; + data?: string | null; checksum: number | bigint; subkey?: string; } From cbf04f8924bbcf3f9491358b260f3ebe916a2e98 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 16:02:41 +0200 Subject: [PATCH 2/6] Comment for old format --- packages/service-core/src/util/protocol-types.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index c0a43ef54..7a54f7729 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -135,6 +135,9 @@ export interface OplogEntry { op: 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; object_type?: string; object_id?: string; + // Note: When clients have both raw_data and binary_data disabled (this only affects legacy + // clients), data is actually a `Record`. Oplog entries are always stored + // as a serialized (JSON) string so that they don't have to be parsed in the sync service. data?: string | null; checksum: number | bigint; subkey?: string; From 4f515022bd991ac197f6bfdc290b2af2c9fa9147 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 16:03:14 +0200 Subject: [PATCH 3/6] Typo --- packages/service-core/src/util/protocol-types.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index 7a54f7729..ce5baa1f6 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -136,8 +136,8 @@ export interface OplogEntry { object_type?: string; object_id?: string; // Note: When clients have both raw_data and binary_data disabled (this only affects legacy - // clients), data is actually a `Record`. Oplog entries are always stored - // as a serialized (JSON) string so that they don't have to be parsed in the sync service. + // clients), data is actually a `Record`. Oplog entries are always + // stored as a serialized (JSON) string so that they don't have to be parsed in the sync service. data?: string | null; checksum: number | bigint; subkey?: string; From 7990e47a6fc0583fb7db7dacfce39b1c80f7acd0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 16:14:18 +0200 Subject: [PATCH 4/6] Use generics to clarify types --- packages/service-core/src/sync/sync.ts | 2 +- .../service-core/src/util/protocol-types.ts | 28 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/packages/service-core/src/sync/sync.ts b/packages/service-core/src/sync/sync.ts index 9ea5fd794..cc8bfea30 100644 --- a/packages/service-core/src/sync/sync.ts +++ b/packages/service-core/src/sync/sync.ts @@ -440,7 +440,7 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator { return { ...bucketData, data: bucketData.data.map((entry) => { diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index ce5baa1f6..d061e16d1 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -1,5 +1,6 @@ import * as t from 'ts-codec'; -import { BucketDescription, BucketPriority } from '@powersync/service-sync-rules'; +import { BucketDescription, BucketPriority, SqliteJsonRow } from '@powersync/service-sync-rules'; +import { JsonContainer } from '@powersync/service-jsonbig'; export const BucketRequest = t.object({ name: t.string, @@ -65,7 +66,7 @@ export interface StreamingSyncCheckpointDiff { } export interface StreamingSyncData { - data: SyncBucketData; + data: SyncBucketData; } export interface StreamingSyncCheckpointComplete { @@ -109,13 +110,9 @@ export interface BucketState { op_id: string; } -export interface SyncDataBatch { - buckets: SyncBucketData[]; -} - -export interface SyncBucketData { +export interface SyncBucketData { bucket: string; - data: OplogEntry[]; + data: OplogEntry[]; /** * True if there _could_ be more data for this bucket, and another request must be made. */ @@ -130,15 +127,20 @@ export interface SyncBucketData { next_after: ProtocolOpId; } -export interface OplogEntry { +export type StoredOplogData = string | null; + +// Note: When clients have both raw_data and binary_data disabled (this only affects legacy +// clients), data is actually a `Record`. Oplog entries are always +// stored as a serialized (JSON) string so that they don't have to be parsed in the sync service, +// this representation only exists on the way out for legacy clients. +export type ProtocolOplogData = SqliteJsonRow | JsonContainer | StoredOplogData; + +export interface OplogEntry { op_id: ProtocolOpId; op: 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; object_type?: string; object_id?: string; - // Note: When clients have both raw_data and binary_data disabled (this only affects legacy - // clients), data is actually a `Record`. Oplog entries are always - // stored as a serialized (JSON) string so that they don't have to be parsed in the sync service. - data?: string | null; + data?: Data; checksum: number | bigint; subkey?: string; } From 100ccec5ff40e84298844290ae601034af28ae2c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 16:16:07 +0200 Subject: [PATCH 5/6] Add changeset entry --- .changeset/beige-clocks-mix.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/beige-clocks-mix.md diff --git a/.changeset/beige-clocks-mix.md b/.changeset/beige-clocks-mix.md new file mode 100644 index 000000000..1b0099006 --- /dev/null +++ b/.changeset/beige-clocks-mix.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-core': patch +--- + +Internal: Improve types for oplog data From 0d58c7b0f6c811697dc935d1b0533b2edabd440e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 May 2025 16:20:13 +0200 Subject: [PATCH 6/6] Fix use in test client --- test-client/src/util.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test-client/src/util.ts b/test-client/src/util.ts index 5c57e1750..51dc4bb3d 100644 --- a/test-client/src/util.ts +++ b/test-client/src/util.ts @@ -1,6 +1,8 @@ import type * as types from '@powersync/service-core'; -export type BucketData = Record; +export type TestOplogEntry = types.OplogEntry; + +export type BucketData = Record; /** * Combine all chunks of received data, excluding any data after the checkpoint. @@ -57,8 +59,8 @@ export function isCheckpoint(line: types.StreamingSyncLine): line is types.Strea * * This is the function $r(B)$, as described in /docs/bucket-properties.md. */ -export function reduceBucket(operations: types.OplogEntry[]) { - let rowState = new Map(); +export function reduceBucket(operations: TestOplogEntry[]) { + let rowState = new Map(); let otherChecksum = 0; for (let op of operations) { @@ -90,7 +92,7 @@ export function reduceBucket(operations: types.OplogEntry[]) { return Number(BigInt(a.op_id) - BigInt(b.op_id)); }); - let finalState: types.OplogEntry[] = [ + let finalState: TestOplogEntry[] = [ // Special operation to indiciate the checksum remainder { op_id: '0', op: 'CLEAR', checksum: otherChecksum }, ...puts @@ -99,7 +101,7 @@ export function reduceBucket(operations: types.OplogEntry[]) { return finalState; } -function rowKey(entry: types.OplogEntry) { +function rowKey(entry: TestOplogEntry) { return `${entry.object_type}/${entry.object_id}/${entry.subkey}`; }