From f678672f9c119a85b2e38d7a46e73ac5c4b5e8a7 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 1 Oct 2025 10:13:43 +0200 Subject: [PATCH 01/11] Compile IR to SQL --- .../electric-db-collection/src/electric.ts | 41 ++++- .../src/sql-compiler.ts | 146 ++++++++++++++++++ 2 files changed, 181 insertions(+), 6 deletions(-) create mode 100644 packages/electric-db-collection/src/sql-compiler.ts diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 3dcb54b64..729517040 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -13,12 +13,14 @@ import { ExpectedNumberInAwaitTxIdError, TimeoutWaitingForTxIdError, } from "./errors" +import { compileSQL } from "./sql-compiler" import type { BaseCollectionConfig, CollectionConfig, DeleteMutationFnParams, Fn, InsertMutationFnParams, + OnLoadMoreOptions, SyncConfig, UpdateMutationFnParams, UtilsRecord, @@ -494,15 +496,42 @@ function createElectricSync>( } }) - // Return the unsubscribe function - return () => { - // Unsubscribe from the stream - unsubscribeStream() - // Abort the abort controller to stop the stream - abortController.abort() + return { + onLoadMore: (opts) => onLoadMore(params, opts), + cleanup: () => { + // Unsubscribe from the stream + unsubscribeStream() + // Abort the abort controller to stop the stream + abortController.abort() + }, } }, // Expose the getSyncMetadata function getSyncMetadata, } } + +async function onLoadMore>( + syncParams: Parameters[`sync`]>[0], + options: OnLoadMoreOptions +) { + const { begin, write, commit } = syncParams + + // TODO: optimize this by keeping track of which snapshot have been loaded already + // and only load this one if it's not a subset of the ones that have been loaded already + + const snapshotParams = compileSQL(options) + + const snapshot = await requestSnapshot(snapshotParams) + + begin() + + snapshot.data.forEach((row) => { + write({ + type: `insert`, + value: row.value, + }) + }) + + commit() +} diff --git a/packages/electric-db-collection/src/sql-compiler.ts b/packages/electric-db-collection/src/sql-compiler.ts new file mode 100644 index 000000000..730859461 --- /dev/null +++ b/packages/electric-db-collection/src/sql-compiler.ts @@ -0,0 +1,146 @@ +import type { IR, OnLoadMoreOptions } from "@tanstack/db" + +export function compileSQL( + options: OnLoadMoreOptions +): ExternalSubsetParamsRecord { + const { where, orderBy, limit } = options + + const params: Array = [] + const compiledSQL: ExternalSubsetParamsRecord = { params } + + if (where) { + // TODO: this only works when the where expression's PropRefs directly reference a column of the collection + // doesn't work if it goes through aliases because then we need to know the entire query to be able to follow the reference until the base collection (cf. followRef function) + compiledSQL.where = compileBasicExpression(where, params) + } + + if (orderBy) { + compiledSQL.orderBy = compileOrderBy(orderBy, params) + } + + if (limit) { + compiledSQL.limit = limit + } + + return compiledSQL +} + +/** + * Compiles the expression to a SQL string and mutates the params array with the values. + * @param exp - The expression to compile + * @param params - The params array + * @returns The compiled SQL string + */ +function compileBasicExpression( + exp: IR.BasicExpression, + params: Array +): string { + switch (exp.type) { + case `val`: + params.push(exp.value) + return `$${params.length}` + case `ref`: + if (exp.path.length !== 1) { + throw new Error( + `Compiler can't handle nested properties: ${exp.path.join(`.`)}` + ) + } + return exp.path[0]! + case `func`: + return compileFunction(exp, params) + } +} + +function compileOrderBy(orderBy: IR.OrderBy, params: Array): string { + const compiledOrderByClauses = orderBy.map((clause: IR.OrderByClause) => + compileOrderByClause(clause, params) + ) + return compiledOrderByClauses.join(`,`) +} + +function compileOrderByClause( + clause: IR.OrderByClause, + params: Array +): string { + // TODO: what to do with stringSort and locale? + // Correctly supporting them is tricky as it depends on Postgres' collation + const { expression, compareOptions } = clause + let sql = compileBasicExpression(expression, params) + + if (compareOptions.direction === `desc`) { + sql = `${sql} DESC` + } + + if (compareOptions.nulls === `first`) { + sql = `${sql} NULLS FIRST` + } + + if (compareOptions.nulls === `last`) { + sql = `${sql} NULLS LAST` + } + + return sql +} + +function compileFunction( + exp: IR.Func, + params: Array = [] +): string { + const { name, args } = exp + + const opName = getOpName(name) + + const compiledArgs = args.map((arg: IR.BasicExpression) => + compileBasicExpression(arg, params) + ) + + if (isBinaryOp(name)) { + if (compiledArgs.length !== 2) { + throw new Error(`Binary operator ${name} expects 2 arguments`) + } + const [lhs, rhs] = compiledArgs + return `${lhs} ${opName} ${rhs}` + } + + return `${opName}(${compiledArgs.join(`,`)})` +} + +function isBinaryOp(name: string): boolean { + const binaryOps = [`eq`, `gt`, `gte`, `lt`, `lte`, `and`, `or`] + return binaryOps.includes(name) +} + +function getOpName(name: string): string { + const opNames = { + eq: `=`, + gt: `>`, + gte: `>=`, + lt: `<`, + lte: `<=`, + add: `+`, + and: `AND`, + or: `OR`, + not: `NOT`, + isUndefined: `IS NULL`, + isNull: `IS NULL`, + in: `IN`, + like: `LIKE`, + ilike: `ILIKE`, + upper: `UPPER`, + lower: `LOWER`, + length: `LENGTH`, + concat: `CONCAT`, + coalesce: `COALESCE`, + } + return opNames[name as keyof typeof opNames] || name +} + +// TODO: remove this type once we rebase on top of Ilia's PR +// that type will be exported by Ilia's PR +export type ExternalSubsetParamsRecord = { + where?: string + params?: Record + limit?: number + offset?: number + orderBy?: string +} From 347a3c633cbd7f3c0a7a9428c9722a5309992482 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 23 Sep 2025 15:21:29 +0200 Subject: [PATCH 02/11] Use the stream's requestSnapshot method --- packages/electric-db-collection/src/electric.ts | 5 +++-- packages/electric-db-collection/src/sql-compiler.ts | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 729517040..f3d47251b 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -497,7 +497,7 @@ function createElectricSync>( }) return { - onLoadMore: (opts) => onLoadMore(params, opts), + onLoadMore: (opts) => onLoadMore(stream, params, opts), cleanup: () => { // Unsubscribe from the stream unsubscribeStream() @@ -512,6 +512,7 @@ function createElectricSync>( } async function onLoadMore>( + stream: ShapeStream, syncParams: Parameters[`sync`]>[0], options: OnLoadMoreOptions ) { @@ -522,7 +523,7 @@ async function onLoadMore>( const snapshotParams = compileSQL(options) - const snapshot = await requestSnapshot(snapshotParams) + const snapshot = await stream.requestSnapshot(snapshotParams) begin() diff --git a/packages/electric-db-collection/src/sql-compiler.ts b/packages/electric-db-collection/src/sql-compiler.ts index 730859461..3d8ee92a7 100644 --- a/packages/electric-db-collection/src/sql-compiler.ts +++ b/packages/electric-db-collection/src/sql-compiler.ts @@ -48,6 +48,8 @@ function compileBasicExpression( return exp.path[0]! case `func`: return compileFunction(exp, params) + default: + throw new Error(`Unknown expression type`) } } From 5a024b57bd84652883a7d86e9c6a99ca3d3f6a1e Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 24 Sep 2025 09:13:10 +0200 Subject: [PATCH 03/11] Remove todo --- packages/electric-db-collection/src/electric.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index f3d47251b..4a4c28f46 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -517,22 +517,15 @@ async function onLoadMore>( options: OnLoadMoreOptions ) { const { begin, write, commit } = syncParams - - // TODO: optimize this by keeping track of which snapshot have been loaded already - // and only load this one if it's not a subset of the ones that have been loaded already - const snapshotParams = compileSQL(options) - const snapshot = await stream.requestSnapshot(snapshotParams) begin() - snapshot.data.forEach((row) => { write({ type: `insert`, value: row.value, }) }) - commit() } From 7dec8d64196e65ad6af1aff35004cb0893e69505 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 24 Sep 2025 09:14:17 +0200 Subject: [PATCH 04/11] Modify output format of SQL compiler and serialize the values into PG string format --- .../src/pg-serializer.ts | 27 +++++++++++++ .../src/sql-compiler.ts | 40 +++++++++++++------ 2 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 packages/electric-db-collection/src/pg-serializer.ts diff --git a/packages/electric-db-collection/src/pg-serializer.ts b/packages/electric-db-collection/src/pg-serializer.ts new file mode 100644 index 000000000..b4a3803a7 --- /dev/null +++ b/packages/electric-db-collection/src/pg-serializer.ts @@ -0,0 +1,27 @@ +export function serialize(value: unknown): string { + if (typeof value === `string`) { + return `'${value}'` + } + + if (value === null || value === undefined) { + return `NULL` + } + + if (typeof value === `boolean`) { + return value ? `true` : `false` + } + + if (value instanceof Date) { + return `'${value.toISOString()}'` + } + + if (Array.isArray(value)) { + return `ARRAY[${value.map(serialize).join(`,`)}]` + } + + if (typeof value === `object`) { + throw new Error(`Cannot serialize object: ${JSON.stringify(value)}`) + } + + return value.toString() +} diff --git a/packages/electric-db-collection/src/sql-compiler.ts b/packages/electric-db-collection/src/sql-compiler.ts index 3d8ee92a7..e76e99887 100644 --- a/packages/electric-db-collection/src/sql-compiler.ts +++ b/packages/electric-db-collection/src/sql-compiler.ts @@ -1,12 +1,18 @@ +import { serialize } from "./pg-serializer" +import type { ExternalSubsetParamsRecord } from "@electric-sql/client" import type { IR, OnLoadMoreOptions } from "@tanstack/db" +export type CompiledSqlRecord = Omit & { + params?: Array +} + export function compileSQL( options: OnLoadMoreOptions ): ExternalSubsetParamsRecord { const { where, orderBy, limit } = options const params: Array = [] - const compiledSQL: ExternalSubsetParamsRecord = { params } + const compiledSQL: CompiledSqlRecord = { params } if (where) { // TODO: this only works when the where expression's PropRefs directly reference a column of the collection @@ -22,7 +28,20 @@ export function compileSQL( compiledSQL.limit = limit } - return compiledSQL + // Serialize the values in the params array into PG formatted strings + // and transform the array into a Record + const paramsRecord = params.reduce( + (acc, param, index) => { + acc[`${index + 1}`] = serialize(param) + return acc + }, + {} as Record + ) + + return { + ...compiledSQL, + params: paramsRecord, + } } /** @@ -134,15 +153,12 @@ function getOpName(name: string): string { concat: `CONCAT`, coalesce: `COALESCE`, } - return opNames[name as keyof typeof opNames] || name -} -// TODO: remove this type once we rebase on top of Ilia's PR -// that type will be exported by Ilia's PR -export type ExternalSubsetParamsRecord = { - where?: string - params?: Record - limit?: number - offset?: number - orderBy?: string + const opName = opNames[name as keyof typeof opNames] + + if (!opName) { + throw new Error(`Unknown operator/function: ${name}`) + } + + return opName } From 61da27713e4e9cbfa5391b0fcb8024a910cbb697 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 1 Oct 2025 10:00:40 +0200 Subject: [PATCH 05/11] Fixes to electric collection + unit test --- .../electric-db-collection/src/electric.ts | 24 +--- .../src/pg-serializer.ts | 10 +- .../src/sql-compiler.ts | 4 +- .../tests/electric-live-query.test.ts | 114 ++++++++++++++++++ 4 files changed, 127 insertions(+), 25 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 4a4c28f46..bcd897dff 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -16,6 +16,7 @@ import { import { compileSQL } from "./sql-compiler" import type { BaseCollectionConfig, + Collection, CollectionConfig, DeleteMutationFnParams, Fn, @@ -497,7 +498,10 @@ function createElectricSync>( }) return { - onLoadMore: (opts) => onLoadMore(stream, params, opts), + onLoadMore: (opts) => { + const snapshotParams = compileSQL(opts) + return stream.requestSnapshot(snapshotParams) + }, cleanup: () => { // Unsubscribe from the stream unsubscribeStream() @@ -511,21 +515,3 @@ function createElectricSync>( } } -async function onLoadMore>( - stream: ShapeStream, - syncParams: Parameters[`sync`]>[0], - options: OnLoadMoreOptions -) { - const { begin, write, commit } = syncParams - const snapshotParams = compileSQL(options) - const snapshot = await stream.requestSnapshot(snapshotParams) - - begin() - snapshot.data.forEach((row) => { - write({ - type: `insert`, - value: row.value, - }) - }) - commit() -} diff --git a/packages/electric-db-collection/src/pg-serializer.ts b/packages/electric-db-collection/src/pg-serializer.ts index b4a3803a7..707c4e1b8 100644 --- a/packages/electric-db-collection/src/pg-serializer.ts +++ b/packages/electric-db-collection/src/pg-serializer.ts @@ -3,6 +3,10 @@ export function serialize(value: unknown): string { return `'${value}'` } + if (typeof value === `number`) { + return value.toString() + } + if (value === null || value === undefined) { return `NULL` } @@ -19,9 +23,5 @@ export function serialize(value: unknown): string { return `ARRAY[${value.map(serialize).join(`,`)}]` } - if (typeof value === `object`) { - throw new Error(`Cannot serialize object: ${JSON.stringify(value)}`) - } - - return value.toString() + throw new Error(`Cannot serialize value: ${JSON.stringify(value)}`) } diff --git a/packages/electric-db-collection/src/sql-compiler.ts b/packages/electric-db-collection/src/sql-compiler.ts index e76e99887..d1d95040e 100644 --- a/packages/electric-db-collection/src/sql-compiler.ts +++ b/packages/electric-db-collection/src/sql-compiler.ts @@ -30,6 +30,7 @@ export function compileSQL( // Serialize the values in the params array into PG formatted strings // and transform the array into a Record + console.log("params", params) const paramsRecord = params.reduce( (acc, param, index) => { acc[`${index + 1}`] = serialize(param) @@ -58,7 +59,8 @@ function compileBasicExpression( case `val`: params.push(exp.value) return `$${params.length}` - case `ref`: + case `ref`: + // TODO: doesn't yet support JSON(B) values which could be accessed with nested props if (exp.path.length !== 1) { throw new Error( `Compiler can't handle nested properties: ${exp.path.join(`.`)}` diff --git a/packages/electric-db-collection/tests/electric-live-query.test.ts b/packages/electric-db-collection/tests/electric-live-query.test.ts index b387f1756..a629677ab 100644 --- a/packages/electric-db-collection/tests/electric-live-query.test.ts +++ b/packages/electric-db-collection/tests/electric-live-query.test.ts @@ -54,10 +54,30 @@ const sampleUsers: Array = [ // Mock the ShapeStream module const mockSubscribe = vi.fn() +const mockRequestSnapshot = vi.fn() const mockStream = { subscribe: mockSubscribe, + requestSnapshot: (...args: any) => { + mockRequestSnapshot(...args) + const results = mockRequestSnapshot.mock.results + const lastResult = results[results.length - 1]!.value + + const subscribers = mockSubscribe.mock.calls.map(args => args[0]) + subscribers.forEach(subscriber => subscriber(lastResult.data.map((row: any) => ({ + type: `insert`, + value: row.value, + key: row.key, + })))) + } } +// Mock the requestSnapshot method +// to return an empty array of data +// since most tests don't use it +mockRequestSnapshot.mockResolvedValue({ + data: [] +}) + vi.mock(`@electric-sql/client`, async () => { const actual = await vi.importActual(`@electric-sql/client`) return { @@ -437,4 +457,98 @@ describe.each([ // Clean up subscription.unsubscribe() }) + if (autoIndex === `eager`) { + it.only(`should load more data via requestSnapshot when creating live query with higher limit`, async () => { + // Reset mocks + vi.clearAllMocks() + mockRequestSnapshot.mockResolvedValue({ + data: [ + { key: 5, value: { id: 5, name: `Eve`, age: 30, email: `eve@example.com`, active: true } }, + { key: 6, value: { id: 6, name: `Frank`, age: 35, email: `frank@example.com`, active: true } }, + ], + }) + + // Initial sync with limited data + simulateInitialSync() + expect(electricCollection.status).toBe(`ready`) + expect(electricCollection.size).toBe(4) + expect(mockRequestSnapshot).toHaveBeenCalledTimes(0) + + // Create first live query with limit of 2 + const limitedLiveQuery = createLiveQueryCollection({ + id: `limited-users-live-query`, + startSync: true, + query: (q) => + q + .from({ user: electricCollection }) + .where(({ user }) => eq(user.active, true)) + .select(({ user }) => ({ + id: user.id, + name: user.name, + active: user.active, + age: user.age, + })) + .orderBy(({ user }) => user.age, `asc`) + .limit(2), + }) + + expect(limitedLiveQuery.status).toBe(`ready`) + expect(limitedLiveQuery.size).toBe(2) // Only first 2 active users + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + + const callArgs = (index: number) => mockRequestSnapshot.mock.calls[index]?.[0] + expect(callArgs(0)).toMatchObject({ + params: { "1": "true" }, + where: "active = $1", + orderBy: "age NULLS FIRST", + limit: 2, + }) + + // Create second live query with higher limit of 5 + const expandedLiveQuery = createLiveQueryCollection({ + id: `expanded-users-live-query`, + startSync: true, + query: (q) => + q + .from({ user: electricCollection }) + .where(({ user }) => eq(user.active, true)) + .select(({ user }) => ({ + id: user.id, + name: user.name, + active: user.active, + })) + .orderBy(({ user }) => user.age, `asc`) + .limit(6), + }) + + // Wait for the live query to process + await new Promise((resolve) => setTimeout(resolve, 0)) + + // Verify that requestSnapshot was called with the correct parameters + expect(mockRequestSnapshot).toHaveBeenCalledTimes(3) + + // Check that first it requested a limit of 6 users + expect(callArgs(1)).toMatchObject({ + params: { "1": "true" }, + where: "active = $1", + orderBy: "age NULLS FIRST", + limit: 6, + }) + + // After this initial snapshot for the new live query it receives all 3 users from the local collection + // so it still needs 3 more users to reach the limit of 6 so it requests 3 more to the sync layer + expect(callArgs(2)).toMatchObject({ + params: { "1": "true", "2": "25" }, + where: "active = $1 AND age > $2", + orderBy: "age NULLS FIRST", + limit: 3, + }) + + // The sync layer won't provide any more users so the DB is exhausted and it stops (i.e. doesn't request more) + + // The expanded live query should now have more data + expect(expandedLiveQuery.status).toBe(`ready`) + expect(expandedLiveQuery.size).toBe(5) // Alice, Bob, Dave from initial + Eve and Frank from additional data + }) + } }) From e3a7d5d0235b3dac674c4374c03369d49d55bfd8 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 29 Sep 2025 11:56:35 +0200 Subject: [PATCH 06/11] Fix unit test for loading more data via requestSnapshot in the Electric collection --- .../electric-db-collection/src/electric.ts | 7 +- .../tests/electric-live-query.test.ts | 104 ++++++++++++------ 2 files changed, 75 insertions(+), 36 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index bcd897dff..019e3f969 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -16,12 +16,10 @@ import { import { compileSQL } from "./sql-compiler" import type { BaseCollectionConfig, - Collection, CollectionConfig, DeleteMutationFnParams, Fn, InsertMutationFnParams, - OnLoadMoreOptions, SyncConfig, UpdateMutationFnParams, UtilsRecord, @@ -498,9 +496,9 @@ function createElectricSync>( }) return { - onLoadMore: (opts) => { + onLoadMore: async (opts) => { const snapshotParams = compileSQL(opts) - return stream.requestSnapshot(snapshotParams) + await stream.requestSnapshot(snapshotParams) }, cleanup: () => { // Unsubscribe from the stream @@ -514,4 +512,3 @@ function createElectricSync>( getSyncMetadata, } } - diff --git a/packages/electric-db-collection/tests/electric-live-query.test.ts b/packages/electric-db-collection/tests/electric-live-query.test.ts index a629677ab..48b5aef04 100644 --- a/packages/electric-db-collection/tests/electric-live-query.test.ts +++ b/packages/electric-db-collection/tests/electric-live-query.test.ts @@ -57,25 +57,34 @@ const mockSubscribe = vi.fn() const mockRequestSnapshot = vi.fn() const mockStream = { subscribe: mockSubscribe, - requestSnapshot: (...args: any) => { - mockRequestSnapshot(...args) - const results = mockRequestSnapshot.mock.results - const lastResult = results[results.length - 1]!.value - - const subscribers = mockSubscribe.mock.calls.map(args => args[0]) - subscribers.forEach(subscriber => subscriber(lastResult.data.map((row: any) => ({ - type: `insert`, + requestSnapshot: async (...args: any) => { + const result = await mockRequestSnapshot(...args) + const subscribers = mockSubscribe.mock.calls.map((args) => args[0]) + const data = [...result.data] + + const messages: Array> = data.map((row: any) => ({ value: row.value, key: row.key, - })))) - } + headers: row.headers, + })) + + if (messages.length > 0) { + // add an up-to-date message + messages.push({ + headers: { control: `up-to-date` }, + }) + } + + subscribers.forEach((subscriber) => subscriber(messages)) + return result + }, } // Mock the requestSnapshot method // to return an empty array of data // since most tests don't use it mockRequestSnapshot.mockResolvedValue({ - data: [] + data: [], }) vi.mock(`@electric-sql/client`, async () => { @@ -458,14 +467,9 @@ describe.each([ subscription.unsubscribe() }) if (autoIndex === `eager`) { - it.only(`should load more data via requestSnapshot when creating live query with higher limit`, async () => { - // Reset mocks - vi.clearAllMocks() + it(`should load more data via requestSnapshot when creating live query with higher limit`, async () => { mockRequestSnapshot.mockResolvedValue({ - data: [ - { key: 5, value: { id: 5, name: `Eve`, age: 30, email: `eve@example.com`, active: true } }, - { key: 6, value: { id: 6, name: `Frank`, age: 35, email: `frank@example.com`, active: true } }, - ], + data: [], }) // Initial sync with limited data @@ -496,15 +500,45 @@ describe.each([ expect(limitedLiveQuery.size).toBe(2) // Only first 2 active users expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) - const callArgs = (index: number) => mockRequestSnapshot.mock.calls[index]?.[0] + const callArgs = (index: number) => + mockRequestSnapshot.mock.calls[index]?.[0] expect(callArgs(0)).toMatchObject({ - params: { "1": "true" }, - where: "active = $1", - orderBy: "age NULLS FIRST", + params: { "1": `true` }, + where: `active = $1`, + orderBy: `age NULLS FIRST`, limit: 2, }) - // Create second live query with higher limit of 5 + // Next call will return a snapshot containing 2 rows + // Calls after that will return the default empty snapshot + mockRequestSnapshot.mockResolvedValueOnce({ + data: [ + { + headers: { operation: `insert` }, + key: 5, + value: { + id: 5, + name: `Eve`, + age: 30, + email: `eve@example.com`, + active: true, + }, + }, + { + headers: { operation: `insert` }, + key: 6, + value: { + id: 6, + name: `Frank`, + age: 35, + email: `frank@example.com`, + active: true, + }, + }, + ], + }) + + // Create second live query with higher limit of 6 const expandedLiveQuery = createLiveQueryCollection({ id: `expanded-users-live-query`, startSync: true, @@ -525,26 +559,34 @@ describe.each([ await new Promise((resolve) => setTimeout(resolve, 0)) // Verify that requestSnapshot was called with the correct parameters - expect(mockRequestSnapshot).toHaveBeenCalledTimes(3) + expect(mockRequestSnapshot).toHaveBeenCalledTimes(4) // Check that first it requested a limit of 6 users expect(callArgs(1)).toMatchObject({ - params: { "1": "true" }, - where: "active = $1", - orderBy: "age NULLS FIRST", + params: { "1": `true` }, + where: `active = $1`, + orderBy: `age NULLS FIRST`, limit: 6, }) // After this initial snapshot for the new live query it receives all 3 users from the local collection // so it still needs 3 more users to reach the limit of 6 so it requests 3 more to the sync layer expect(callArgs(2)).toMatchObject({ - params: { "1": "true", "2": "25" }, - where: "active = $1 AND age > $2", - orderBy: "age NULLS FIRST", + params: { "1": `true`, "2": `25` }, + where: `active = $1 AND age > $2`, + orderBy: `age NULLS FIRST`, limit: 3, }) - // The sync layer won't provide any more users so the DB is exhausted and it stops (i.e. doesn't request more) + // The previous snapshot returned 2 more users so it still needs 1 more user to reach the limit of 6 + expect(callArgs(3)).toMatchObject({ + params: { "1": `true`, "2": `35` }, + where: `active = $1 AND age > $2`, + orderBy: `age NULLS FIRST`, + limit: 1, + }) + + // The sync layer won't provide any more users so the DB is exhausted and it stops (i.e. doesn't request more) // The expanded live query should now have more data expect(expandedLiveQuery.status).toBe(`ready`) From 310bd6599034a99e66be3a1ec7236165e94c0cc9 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 1 Oct 2025 10:02:58 +0200 Subject: [PATCH 07/11] Remove debug logging in electric collection --- packages/electric-db-collection/src/sql-compiler.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/electric-db-collection/src/sql-compiler.ts b/packages/electric-db-collection/src/sql-compiler.ts index d1d95040e..92ef7d287 100644 --- a/packages/electric-db-collection/src/sql-compiler.ts +++ b/packages/electric-db-collection/src/sql-compiler.ts @@ -30,7 +30,6 @@ export function compileSQL( // Serialize the values in the params array into PG formatted strings // and transform the array into a Record - console.log("params", params) const paramsRecord = params.reduce( (acc, param, index) => { acc[`${index + 1}`] = serialize(param) @@ -59,7 +58,7 @@ function compileBasicExpression( case `val`: params.push(exp.value) return `$${params.length}` - case `ref`: + case `ref`: // TODO: doesn't yet support JSON(B) values which could be accessed with nested props if (exp.path.length !== 1) { throw new Error( From 6df040c4556a7104c1df4e49b1b5ee9e3e272b98 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 6 Oct 2025 15:57:20 +0200 Subject: [PATCH 08/11] Update type name --- packages/electric-db-collection/src/sql-compiler.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/electric-db-collection/src/sql-compiler.ts b/packages/electric-db-collection/src/sql-compiler.ts index 92ef7d287..421c2ab7c 100644 --- a/packages/electric-db-collection/src/sql-compiler.ts +++ b/packages/electric-db-collection/src/sql-compiler.ts @@ -1,14 +1,12 @@ import { serialize } from "./pg-serializer" -import type { ExternalSubsetParamsRecord } from "@electric-sql/client" +import type { SubsetParams } from "@electric-sql/client" import type { IR, OnLoadMoreOptions } from "@tanstack/db" -export type CompiledSqlRecord = Omit & { +export type CompiledSqlRecord = Omit & { params?: Array } -export function compileSQL( - options: OnLoadMoreOptions -): ExternalSubsetParamsRecord { +export function compileSQL(options: OnLoadMoreOptions): SubsetParams { const { where, orderBy, limit } = options const params: Array = [] From 3f6dc70579a95cf5613a93e4e0d452c009f871c0 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 6 Oct 2025 15:57:39 +0200 Subject: [PATCH 09/11] Upgrade electric client version --- pnpm-lock.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 88dc1cb8f..a37082d80 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1372,6 +1372,9 @@ packages: '@electric-sql/client@1.0.14': resolution: {integrity: sha512-LtPAfeMxXRiYS0hyDQ5hue2PjljUiK9stvzsVyVb4nwxWQxfOWTSF42bHTs/o5i3x1T4kAQ7mwHpxa4A+f8X7Q==} + '@electric-sql/client@1.0.14': + resolution: {integrity: sha512-LtPAfeMxXRiYS0hyDQ5hue2PjljUiK9stvzsVyVb4nwxWQxfOWTSF42bHTs/o5i3x1T4kAQ7mwHpxa4A+f8X7Q==} + '@emnapi/core@1.5.0': resolution: {integrity: sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==} @@ -9288,6 +9291,12 @@ snapshots: '@drizzle-team/brocli@0.10.2': {} + '@electric-sql/client@1.0.14': + dependencies: + '@microsoft/fetch-event-source': 2.0.1 + optionalDependencies: + '@rollup/rollup-darwin-arm64': 4.50.1 + '@electric-sql/client@1.0.14': dependencies: '@microsoft/fetch-event-source': 2.0.1 From dd337d8bb2910501b928a6bf158acc9320a624f9 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Mon, 6 Oct 2025 15:59:34 +0200 Subject: [PATCH 10/11] Changeset --- .changeset/tender-carpets-cheat.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tender-carpets-cheat.md diff --git a/.changeset/tender-carpets-cheat.md b/.changeset/tender-carpets-cheat.md new file mode 100644 index 000000000..77c9dfd73 --- /dev/null +++ b/.changeset/tender-carpets-cheat.md @@ -0,0 +1,5 @@ +--- +"@tanstack/electric-db-collection": patch +--- + +Handle predicates that are pushed down. From a118a66ee2efb5d308f66cf4bf53b3a1d5a5552c Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 7 Oct 2025 12:24:41 +0200 Subject: [PATCH 11/11] Update lockfile --- pnpm-lock.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a37082d80..88dc1cb8f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1372,9 +1372,6 @@ packages: '@electric-sql/client@1.0.14': resolution: {integrity: sha512-LtPAfeMxXRiYS0hyDQ5hue2PjljUiK9stvzsVyVb4nwxWQxfOWTSF42bHTs/o5i3x1T4kAQ7mwHpxa4A+f8X7Q==} - '@electric-sql/client@1.0.14': - resolution: {integrity: sha512-LtPAfeMxXRiYS0hyDQ5hue2PjljUiK9stvzsVyVb4nwxWQxfOWTSF42bHTs/o5i3x1T4kAQ7mwHpxa4A+f8X7Q==} - '@emnapi/core@1.5.0': resolution: {integrity: sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==} @@ -9291,12 +9288,6 @@ snapshots: '@drizzle-team/brocli@0.10.2': {} - '@electric-sql/client@1.0.14': - dependencies: - '@microsoft/fetch-event-source': 2.0.1 - optionalDependencies: - '@rollup/rollup-darwin-arm64': 4.50.1 - '@electric-sql/client@1.0.14': dependencies: '@microsoft/fetch-event-source': 2.0.1