diff --git a/packages/db-ivm/src/index.ts b/packages/db-ivm/src/index.ts index 3ca1f26c6..cae148b46 100644 --- a/packages/db-ivm/src/index.ts +++ b/packages/db-ivm/src/index.ts @@ -2,4 +2,4 @@ export * from './d2.js' export * from './multiset.js' export * from './operators/index.js' export * from './types.js' -export { compareKeys } from './utils.js' +export { compareKeys, serializeValue } from './utils.js' diff --git a/packages/db-ivm/src/operators/groupBy.ts b/packages/db-ivm/src/operators/groupBy.ts index b752ae560..24b884715 100644 --- a/packages/db-ivm/src/operators/groupBy.ts +++ b/packages/db-ivm/src/operators/groupBy.ts @@ -1,3 +1,4 @@ +import { serializeValue } from '../utils.js' import { map } from './map.js' import { reduce } from './reduce.js' import type { IStreamBuilder, KeyValue } from '../types.js' @@ -67,7 +68,7 @@ export function groupBy< const withKeysAndValues = stream.pipe( map((data) => { const key = keyExtractor(data) - const keyString = JSON.stringify(key) + const keyString = serializeValue(key) // Create values object with pre-aggregated values const values: Record = {} diff --git a/packages/db-ivm/src/utils.ts b/packages/db-ivm/src/utils.ts index 8f931c362..70cfda48c 100644 --- a/packages/db-ivm/src/utils.ts +++ b/packages/db-ivm/src/utils.ts @@ -192,3 +192,20 @@ export function compareKeys(a: string | number, b: string | number): number { // Different types: strings come before numbers return typeof a === `string` ? -1 : 1 } + +/** + * Serializes a value for use as a key, handling BigInt and Date values that JSON.stringify cannot handle. + * Uses JSON.stringify with a replacer function to convert BigInt values to strings and Date values to ISO strings. + * This is used for creating string keys in groupBy operations. + */ +export function serializeValue(value: unknown): string { + return JSON.stringify(value, (_, val) => { + if (typeof val === 'bigint') { + return val.toString() + } + if (val instanceof Date) { + return val.toISOString() + } + return val + }) +} diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index e9c8fa436..9bfd327c0 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -1,4 +1,10 @@ -import { filter, groupBy, groupByOperators, map } from '@tanstack/db-ivm' +import { + filter, + groupBy, + groupByOperators, + map, + serializeValue, +} from '@tanstack/db-ivm' import { Func, PropRef, getHavingExpression } from '../ir.js' import { AggregateFunctionNotInSelectError, @@ -248,7 +254,7 @@ export function processGroupBy( for (let i = 0; i < groupByClause.length; i++) { keyParts.push(aggregatedRow[`__key_${i}`]) } - finalKey = JSON.stringify(keyParts) + finalKey = serializeValue(keyParts) } return [ diff --git a/packages/db/tests/query/group-by.test.ts b/packages/db/tests/query/group-by.test.ts index e21f7be2c..0785e8a8c 100644 --- a/packages/db/tests/query/group-by.test.ts +++ b/packages/db/tests/query/group-by.test.ts @@ -1108,6 +1108,345 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { expect(customer1?.min_quantity).toBe(1) expect(customer1?.max_quantity).toBe(2) }) + + test(`group by BigInt column should not throw JSON.stringify error`, () => { + // Define a type with BigInt field (simulating a Postgres bigint column) + type District = { + id: number + district_id: bigint + name: string + status: string + } + + // Use BigInt values beyond MAX_SAFE_INTEGER to test BigInt serialization + const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER + 1) + + const sampleDistricts: Array = [ + { + id: 1, + district_id: MAX_SAFE_BIGINT, + name: 'District A', + status: 'active', + }, + { + id: 2, + district_id: MAX_SAFE_BIGINT + 1n, + name: 'District B', + status: 'active', + }, + { + id: 3, + district_id: MAX_SAFE_BIGINT, // Same as first + name: 'District C', + status: 'inactive', + }, + { + id: 4, + district_id: MAX_SAFE_BIGINT + 2n, + name: 'District D', + status: 'active', + }, + ] + + const districtsCollection = createCollection( + mockSyncCollectionOptions({ + id: `test-districts`, + getKey: (district) => district.id, + initialData: sampleDistricts, + autoIndex, + }), + ) + + // This should not throw "Do not know how to serialize a BigInt" error + const districtSummary = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ districts: districtsCollection }) + .groupBy(({ districts }) => districts.district_id) + .select(({ districts }) => ({ + district_id: districts.district_id, + count: count(districts.id), + })), + }) + + // Should have 3 groups (two districts share the same district_id) + expect(districtSummary.size).toBe(3) + + // Check the group with MAX_SAFE_BIGINT + const group1 = Array.from(districtSummary.values()).find( + (d) => d.district_id === MAX_SAFE_BIGINT, + ) + expect(group1).toBeDefined() + expect(group1?.count).toBe(2) // Districts 1 and 3 + + // Check the group with MAX_SAFE_BIGINT + 1n + const group2 = Array.from(districtSummary.values()).find( + (d) => d.district_id === MAX_SAFE_BIGINT + 1n, + ) + expect(group2).toBeDefined() + expect(group2?.count).toBe(1) // District 2 + + // Check the group with MAX_SAFE_BIGINT + 2n + const group3 = Array.from(districtSummary.values()).find( + (d) => d.district_id === MAX_SAFE_BIGINT + 2n, + ) + expect(group3).toBeDefined() + expect(group3?.count).toBe(1) // District 4 + }) + + test(`group by Date column should serialize correctly`, () => { + type Event = { + id: number + event_date: Date + name: string + category: string + } + + const date1 = new Date('2023-01-15T10:00:00Z') + const date2 = new Date('2023-01-20T14:30:00Z') + const date3 = new Date('2023-01-15T10:00:00Z') // Same as date1 + + const sampleEvents: Array = [ + { + id: 1, + event_date: date1, + name: 'Event A', + category: 'conference', + }, + { + id: 2, + event_date: date2, + name: 'Event B', + category: 'workshop', + }, + { + id: 3, + event_date: date3, // Same date as Event A + name: 'Event C', + category: 'conference', + }, + { + id: 4, + event_date: new Date('2023-02-01T09:00:00Z'), + name: 'Event D', + category: 'workshop', + }, + ] + + const eventsCollection = createCollection( + mockSyncCollectionOptions({ + id: `test-events`, + getKey: (event) => event.id, + initialData: sampleEvents, + autoIndex, + }), + ) + + // Group by Date - should serialize dates to ISO strings for keys + const eventSummary = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ events: eventsCollection }) + .groupBy(({ events }) => events.event_date) + .select(({ events }) => ({ + event_date: events.event_date, + count: count(events.id), + })), + }) + + // Should have 3 groups (two events share the same date) + expect(eventSummary.size).toBe(3) + + // Check the group with date1 (which equals date3) + const group1 = Array.from(eventSummary.values()).find( + (e) => e.event_date.getTime() === date1.getTime(), + ) + expect(group1).toBeDefined() + expect(group1?.count).toBe(2) // Events 1 and 3 + + // Check the group with date2 + const group2 = Array.from(eventSummary.values()).find( + (e) => e.event_date.getTime() === date2.getTime(), + ) + expect(group2).toBeDefined() + expect(group2?.count).toBe(1) // Event 2 + + // Check the group with the February date + const group3 = Array.from(eventSummary.values()).find( + (e) => + e.event_date.getTime() === + new Date('2023-02-01T09:00:00Z').getTime(), + ) + expect(group3).toBeDefined() + expect(group3?.count).toBe(1) // Event 4 + }) + + test(`group by multiple columns including Date should not throw error`, () => { + type LogEntry = { + id: number + timestamp: Date + level: string + service: string + } + + const timestamp1 = new Date('2023-01-15T10:00:00Z') + const timestamp2 = new Date('2023-01-15T10:00:00Z') // Same as timestamp1 + + const sampleLogs: Array = [ + { + id: 1, + timestamp: timestamp1, + level: 'error', + service: 'api', + }, + { + id: 2, + timestamp: timestamp1, + level: 'info', + service: 'api', + }, + { + id: 3, + timestamp: timestamp2, + level: 'error', + service: 'worker', + }, + { + id: 4, + timestamp: new Date('2023-01-20T14:30:00Z'), + level: 'warning', + service: 'api', + }, + ] + + const logsCollection = createCollection( + mockSyncCollectionOptions({ + id: `test-logs`, + getKey: (log) => log.id, + initialData: sampleLogs, + autoIndex, + }), + ) + + // Group by both Date and string - should not throw + const logSummary = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ logs: logsCollection }) + .groupBy(({ logs }) => [logs.timestamp, logs.level]) + .select(({ logs }) => ({ + timestamp: logs.timestamp, + level: logs.level, + count: count(logs.id), + })), + }) + + expect(logSummary.size).toBe(3) // Three distinct combinations + + // Find the group with timestamp1 and 'error' + const group1 = Array.from(logSummary.values()).find( + (l) => + l.timestamp.getTime() === timestamp1.getTime() && + l.level === 'error', + ) + expect(group1).toBeDefined() + expect(group1?.count).toBe(2) // Logs 1 and 3 (same timestamp, same level) + + // Find the group with timestamp1 and 'info' + const group2 = Array.from(logSummary.values()).find( + (l) => + l.timestamp.getTime() === timestamp1.getTime() && + l.level === 'info', + ) + expect(group2).toBeDefined() + expect(group2?.count).toBe(1) // Log 2 + + // Find the group with the different timestamp and 'warning' + const group3 = Array.from(logSummary.values()).find( + (l) => + l.timestamp.getTime() === + new Date('2023-01-20T14:30:00Z').getTime() && + l.level === 'warning', + ) + expect(group3).toBeDefined() + expect(group3?.count).toBe(1) // Log 4 + }) + + test(`group by multiple columns including BigInt should not throw error`, () => { + type School = { + id: number + district_id: bigint + name: string + status: string + } + + // Use BigInt values beyond MAX_SAFE_INTEGER to test BigInt serialization + const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER + 1) + + const sampleSchools: Array = [ + { + id: 1, + district_id: MAX_SAFE_BIGINT, + name: 'School A', + status: 'active', + }, + { + id: 2, + district_id: MAX_SAFE_BIGINT, + name: 'School B', + status: 'active', + }, + { + id: 3, + district_id: MAX_SAFE_BIGINT + 1n, + name: 'School C', + status: 'inactive', + }, + ] + + const schoolsCollection = createCollection( + mockSyncCollectionOptions({ + id: `test-schools`, + getKey: (school) => school.id, + initialData: sampleSchools, + autoIndex, + }), + ) + + // Group by both BigInt and string - should not throw + const schoolSummary = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ schools: schoolsCollection }) + .groupBy(({ schools }) => [schools.district_id, schools.status]) + .select(({ schools }) => ({ + district_id: schools.district_id, + status: schools.status, + count: count(schools.id), + })), + }) + + expect(schoolSummary.size).toBe(2) // Two distinct combinations + + // Find the group with MAX_SAFE_BIGINT and 'active' + const group1 = Array.from(schoolSummary.values()).find( + (s) => s.district_id === MAX_SAFE_BIGINT && s.status === 'active', + ) + expect(group1).toBeDefined() + expect(group1?.count).toBe(2) // Schools 1 and 2 + + // Find the group with MAX_SAFE_BIGINT + 1n and 'inactive' + const group2 = Array.from(schoolSummary.values()).find( + (s) => + s.district_id === MAX_SAFE_BIGINT + 1n && s.status === 'inactive', + ) + expect(group2).toBeDefined() + expect(group2?.count).toBe(1) // School 3 + }) }) describe(`Nested Object GroupBy`, () => {