Skip to content

Commit 6c8ec64

Browse files
samwillissamwillis
committed
checkpoint
1 parent f163eb4 commit 6c8ec64

File tree

10 files changed

+158
-34
lines changed

10 files changed

+158
-34
lines changed

‎packages/db-collection-e2e/src/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ export * from "./fixtures/seed-data"
1010
export * from "./utils/helpers"
1111
export * from "./utils/assertions"
1212

13+
// Export specific utilities for convenience
14+
export { waitFor, waitForQueryData, waitForCollectionSize } from "./utils/helpers"
15+
1316
// Export test suite creators
1417
export { createPredicatesTestSuite } from "./suites/predicates.suite"
1518
export { createPaginationTestSuite } from "./suites/pagination.suite"

‎packages/db-collection-e2e/src/suites/deduplication.suite.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import { describe, expect, it } from "vitest"
88
import { createLiveQueryCollection, eq, gt, isNull, lt } from "@tanstack/db"
9+
import { waitForQueryData } from "../utils/helpers"
910
import type { E2ETestConfig } from "../types"
1011

1112
export function createDeduplicationTestSuite(
@@ -208,6 +209,8 @@ export function createDeduplicationTestSuite(
208209
)
209210

210211
await Promise.all([query1.preload(), query2.preload()])
212+
await waitForQueryData(query1, { minSize: 10 })
213+
await waitForQueryData(query2, { minSize: 20 })
211214

212215
expect(query1.size).toBe(10)
213216
expect(query2.size).toBe(20)
@@ -234,6 +237,8 @@ export function createDeduplicationTestSuite(
234237
)
235238

236239
await Promise.all([query1.preload(), query2.preload()])
240+
await waitForQueryData(query1, { minSize: 10 })
241+
await waitForQueryData(query2, { minSize: 10 })
237242

238243
expect(query1.size).toBe(10)
239244
expect(query2.size).toBe(10)

‎packages/db-collection-e2e/src/suites/joins.suite.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import { describe, expect, it } from "vitest"
88
import { createLiveQueryCollection, eq, gt, isNull } from "@tanstack/db"
9+
import { waitFor, waitForQueryData } from "../utils/helpers"
910
import type { E2ETestConfig } from "../types"
1011

1112
export function createJoinsTestSuite(getConfig: () => Promise<E2ETestConfig>) {
@@ -86,6 +87,8 @@ export function createJoinsTestSuite(getConfig: () => Promise<E2ETestConfig>) {
8687
)
8788

8889
await query.preload()
90+
// Joins with on-demand collections may need more time to load data from multiple sources
91+
await waitForQueryData(query, { minSize: 1, timeout: 5000 })
8992

9093
const results = Array.from(query.state.values())
9194
expect(results.length).toBeGreaterThan(0)
@@ -113,14 +116,28 @@ export function createJoinsTestSuite(getConfig: () => Promise<E2ETestConfig>) {
113116

114117
await query.preload()
115118

119+
// For joins with ordering, we need to wait for sufficient data in BOTH collections
120+
// Wait for the posts collection to load enough data (not just the query results)
121+
await waitFor(() => postsCollection.size >= 100, {
122+
timeout: 5000,
123+
interval: 50,
124+
message: `Posts collection did not fully load (got ${postsCollection.size}/100)`,
125+
})
126+
127+
// Also wait for query to have data
128+
await waitForQueryData(query, { minSize: 50, timeout: 5000 })
129+
116130
const results = Array.from(query.state.values())
117131
expect(results.length).toBeGreaterThan(0)
118132

119-
// Verify sorting by viewCount
133+
// All results MUST have viewCount field (verifies join completed successfully)
134+
expect(results.every((r) => typeof r.viewCount === `number`)).toBe(true)
135+
136+
// Verify sorting by viewCount (descending)
120137
for (let i = 1; i < results.length; i++) {
121-
expect(results[i - 1]!.viewCount).toBeGreaterThanOrEqual(
122-
results[i]!.viewCount
123-
)
138+
const prevCount = results[i - 1]!.viewCount
139+
const currCount = results[i]!.viewCount
140+
expect(prevCount).toBeGreaterThanOrEqual(currCount)
124141
}
125142
})
126143

@@ -233,6 +250,8 @@ export function createJoinsTestSuite(getConfig: () => Promise<E2ETestConfig>) {
233250
)
234251

235252
await query.preload()
253+
// 3-way joins with on-demand collections need more time
254+
await waitForQueryData(query, { minSize: 1, timeout: 5000 })
236255

237256
const results = Array.from(query.state.values())
238257
expect(results.length).toBeGreaterThan(0)

‎packages/db-collection-e2e/src/suites/pagination.suite.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
assertCollectionSize,
1212
assertSorted,
1313
} from "../utils/assertions"
14+
import { waitForQueryData } from "../utils/helpers"
1415
import type { E2ETestConfig } from "../types"
1516

1617
export function createPaginationTestSuite(
@@ -100,6 +101,7 @@ export function createPaginationTestSuite(
100101
)
101102

102103
await query.preload()
104+
await waitForQueryData(query, { minSize: 1 })
103105

104106
const results = Array.from(query.state.values())
105107
expect(results.length).toBeGreaterThan(0)
@@ -130,6 +132,7 @@ export function createPaginationTestSuite(
130132
)
131133

132134
await query.preload()
135+
await waitForQueryData(query, { minSize: 10 })
133136

134137
assertCollectionSize(query, 10)
135138
})
@@ -179,6 +182,7 @@ export function createPaginationTestSuite(
179182
)
180183

181184
await query.preload()
185+
await waitForQueryData(query, { minSize: 5 })
182186

183187
assertCollectionSize(query, 5)
184188
const results = Array.from(query.state.values())
@@ -200,6 +204,7 @@ export function createPaginationTestSuite(
200204
)
201205

202206
await query.preload()
207+
await waitForQueryData(query, { minSize: 80 })
203208

204209
const results = Array.from(query.state.values())
205210
expect(results.length).toBe(80) // 100 - 20 = 80
@@ -276,6 +281,7 @@ export function createPaginationTestSuite(
276281
)
277282

278283
await query.preload()
284+
await waitForQueryData(query, { minSize: 1 })
279285

280286
expect(query.size).toBeLessThanOrEqual(10)
281287
expect(query.size).toBeGreaterThan(0)
@@ -313,6 +319,7 @@ export function createPaginationTestSuite(
313319
)
314320

315321
await query.preload()
322+
await waitForQueryData(query, { minSize: 10 })
316323

317324
// Verify we got exactly 10 records
318325
assertCollectionSize(query, 10)

‎packages/db-collection-e2e/src/suites/predicates.suite.ts‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
or,
2121
} from "@tanstack/db"
2222
import { assertAllItemsMatch, assertCollectionSize } from "../utils/assertions"
23+
import { waitForQueryData } from "../utils/helpers"
2324
import type { E2ETestConfig } from "../types"
2425

2526
export function createPredicatesTestSuite(
@@ -38,6 +39,9 @@ export function createPredicatesTestSuite(
3839
)
3940

4041
await query.preload()
42+
43+
// Wait for Electric to stream snapshot data (on-demand collections load async)
44+
await waitForQueryData(query, { minSize: 1 })
4145

4246
const results = Array.from(query.state.values())
4347
expect(results.length).toBeGreaterThan(0)
@@ -70,6 +74,7 @@ export function createPredicatesTestSuite(
7074
)
7175

7276
await query.preload()
77+
await waitForQueryData(query, { minSize: 1 })
7378

7479
const results = Array.from(query.state.values())
7580
expect(results.length).toBeGreaterThan(0)
@@ -80,7 +85,7 @@ export function createPredicatesTestSuite(
8085
const config = await getConfig()
8186
const usersCollection = config.collections.onDemand.users
8287

83-
const testUserId = `user-0000-4000-8000-000000000000`
88+
const testUserId = `00000000-0000-4000-8000-000000000000` // User ID for index 0
8489

8590
const query = createLiveQueryCollection((q) =>
8691
q
@@ -89,6 +94,7 @@ export function createPredicatesTestSuite(
8994
)
9095

9196
await query.preload()
97+
await waitForQueryData(query, { minSize: 1 })
9298

9399
assertCollectionSize(query, 1)
94100
const result = Array.from(query.state.values())[0]
@@ -106,6 +112,7 @@ export function createPredicatesTestSuite(
106112
)
107113

108114
await query.preload()
115+
await waitForQueryData(query, { minSize: 1 })
109116

110117
const results = Array.from(query.state.values())
111118
expect(results.length).toBeGreaterThan(0)
@@ -125,6 +132,7 @@ export function createPredicatesTestSuite(
125132
)
126133

127134
await query.preload()
135+
await waitForQueryData(query, { minSize: 1 })
128136

129137
const results = Array.from(query.state.values())
130138
expect(results.length).toBeGreaterThan(0)
@@ -142,6 +150,7 @@ export function createPredicatesTestSuite(
142150
)
143151

144152
await query.preload()
153+
await waitForQueryData(query, { minSize: 1 })
145154

146155
const results = Array.from(query.state.values())
147156
expect(results.length).toBeGreaterThan(0)
@@ -266,9 +275,9 @@ export function createPredicatesTestSuite(
266275
const usersCollection = config.collections.onDemand.users
267276

268277
const userIds = [
269-
`user-0000-4000-8000-000000000000`,
270-
`user-0000-4000-8000-000000000001`,
271-
`user-0000-4000-8000-000000000002`,
278+
`00000000-0000-4000-8000-000000000000`, // User ID for index 0
279+
`00000001-0000-4000-8000-000000000001`, // User ID for index 1
280+
`00000002-0000-4000-8000-000000000002`, // User ID for index 2
272281
]
273282

274283
const query = createLiveQueryCollection((q) =>
@@ -311,6 +320,7 @@ export function createPredicatesTestSuite(
311320
)
312321

313322
await query.preload()
323+
await waitForQueryData(query, { minSize: 1 })
314324

315325
const results = Array.from(query.state.values())
316326
expect(results.length).toBeGreaterThan(0)
@@ -328,6 +338,7 @@ export function createPredicatesTestSuite(
328338
)
329339

330340
await query.preload()
341+
await waitForQueryData(query, { minSize: 1 })
331342

332343
const results = Array.from(query.state.values())
333344
expect(results.length).toBeGreaterThan(0)
@@ -345,6 +356,7 @@ export function createPredicatesTestSuite(
345356
)
346357

347358
await query.preload()
359+
await waitForQueryData(query, { minSize: 1 })
348360

349361
const results = Array.from(query.state.values())
350362
expect(results.length).toBeGreaterThan(0)
@@ -434,6 +446,7 @@ export function createPredicatesTestSuite(
434446
)
435447

436448
await query.preload()
449+
await waitForQueryData(query, { minSize: 1 })
437450

438451
// Verify that the underlying collection didn't load ALL users
439452
// In on-demand mode, it should only load age=25 users
@@ -453,6 +466,7 @@ export function createPredicatesTestSuite(
453466
)
454467

455468
await query.preload()
469+
await waitForQueryData(query, { minSize: 1 })
456470

457471
const results = Array.from(query.state.values())
458472
expect(results.length).toBeGreaterThan(0)

‎packages/db-collection-e2e/src/utils/helpers.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,30 @@ export async function waitForCollectionReady<T extends object>(
114114
})
115115
}
116116

117+
/**
118+
* Wait for a query to have data after preload.
119+
* This is necessary for on-demand collections where Electric streams
120+
* snapshot data asynchronously after loadSubset is triggered.
121+
*/
122+
export async function waitForQueryData<T extends object>(
123+
query: Collection<T>,
124+
options: {
125+
minSize?: number
126+
timeout?: number
127+
} = {}
128+
): Promise<void> {
129+
const { minSize = 1, timeout = 2000 } = options
130+
131+
await waitFor(
132+
() => query.size >= minSize,
133+
{
134+
timeout,
135+
interval: 10,
136+
message: `Query did not load data (expected >= ${minSize}, got ${query.size})`,
137+
}
138+
)
139+
}
140+
117141
/**
118142
* Create a deduplication counter for testing
119143
*/

‎packages/electric-db-collection/e2e/electric.e2e.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ describe(`Electric Collection E2E Tests`, () => {
8383
`)
8484

8585
// Insert seed data
86+
console.log(`Inserting ${seedData.users.length} users...`)
8687
for (const user of seedData.users) {
8788
await dbClient.query(
8889
`INSERT INTO ${usersTable} (id, name, email, age, "isActive", "createdAt", metadata, "deletedAt")
@@ -99,6 +100,7 @@ describe(`Electric Collection E2E Tests`, () => {
99100
]
100101
)
101102
}
103+
console.log(`Inserted ${seedData.users.length} users successfully`)
102104

103105
for (const post of seedData.posts) {
104106
await dbClient.query(

‎packages/electric-db-collection/src/pg-serializer.ts‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,56 @@
1+
/**
2+
* Serialize values for Electric SQL subset parameters.
3+
*
4+
* IMPORTANT: Electric expects RAW values, NOT SQL-formatted literals.
5+
* Electric handles all type casting and escaping on the server side.
6+
* The params Record<string, string> contains the actual values as strings,
7+
* and Electric will parse/cast them based on the column type in the WHERE clause.
8+
*
9+
* @param value - The value to serialize
10+
* @returns The raw value as a string (no SQL formatting/quoting)
11+
*/
112
export function serialize(value: unknown): string {
13+
// Handle null/undefined - return empty string (Electric interprets as NULL in typed context)
14+
if (value === null || value === undefined) {
15+
return ``
16+
}
17+
18+
// Handle strings - return as-is (NO quotes, Electric handles escaping)
219
if (typeof value === `string`) {
3-
return `'${value}'`
20+
return value
421
}
522

23+
// Handle numbers - convert to string
624
if (typeof value === `number`) {
725
return value.toString()
826
}
927

10-
if (value === null || value === undefined) {
11-
return `NULL`
12-
}
13-
28+
// Handle booleans - return as lowercase string
1429
if (typeof value === `boolean`) {
1530
return value ? `true` : `false`
1631
}
1732

33+
// Handle dates - return ISO format (NO quotes)
1834
if (value instanceof Date) {
19-
return `'${value.toISOString()}'`
35+
return value.toISOString()
2036
}
2137

38+
// Handle arrays - for = ANY() operator, serialize as Postgres array literal
39+
// Format: {val1,val2,val3} with proper escaping
2240
if (Array.isArray(value)) {
23-
return `ARRAY[${value.map(serialize).join(`,`)}]`
41+
// Postgres array literal format uses curly braces
42+
const elements = value.map((item) => {
43+
if (item === null || item === undefined) {
44+
return `NULL`
45+
}
46+
if (typeof item === `string`) {
47+
// Escape quotes and backslashes for Postgres array literals
48+
const escaped = item.replace(/\\/g, `\\\\`).replace(/"/g, `\\"`)
49+
return `"${escaped}"`
50+
}
51+
return serialize(item)
52+
})
53+
return `{${elements.join(`,`)}}`
2454
}
2555

2656
throw new Error(`Cannot serialize value: ${JSON.stringify(value)}`)

0 commit comments

Comments
 (0)