Custom json handling (#481)

Author: looskie

examples/README.md

@@ -22,6 +22,7 @@ If something is missing, or you found a mistake in one of these examples, please
 - [read_only_user.ts](read_only_user.ts) - an example of using the client with a read-only user, with possible read-only user limitations highlights.
 - [basic_tls.ts](node/basic_tls.ts) - (Node.js only) using certificates for basic TLS authentication.
 - [mutual_tls.ts](node/mutual_tls.ts) - (Node.js only) using certificates for mutual TLS authentication.
+- [custom_json_handling.ts](custom_json_handling.ts) - Customize JSON serialization/deserialization by providing a custom `parse` and `stringify` function. This is particularly useful when working with obscure data formats like `bigint`s.
 
 #### Creating tables
 

examples/custom_json_handling.ts

@@ -0,0 +1,65 @@
+import { createClient } from '@clickhouse/client' // or '@clickhouse/client-web'
+
+/**
+ * Similar to `examples/insert_js_dates.ts` but testing custom JSON handling
+ *
+ * JSON.stringify does not handle BigInt data types by default, so we'll provide
+ * a custom serializer before passing it to the JSON.stringify function.
+ *
+ * This example also shows how you can serialize Date objects in a custom way.
+ */
+void (async () => {
+  const valueSerializer = (value: unknown): unknown => {
+    if (value instanceof Date) {
+      // if you would have put this in the `replacer` parameter of JSON.stringify, (e.x: JSON.stringify(obj, replacerFn))
+      // it would have been an ISO string, but since we are serializing before `stringify`ing,
+      // it will convert it before the `.toJSON()` method has been called
+      return value.getTime()
+    }
+
+    if (typeof value === 'bigint') {
+      return value.toString()
+    }
+
+    if (Array.isArray(value)) {
+      return value.map(valueSerializer)
+    }
+
+    return value
+  }
+
+  const tableName = 'inserts_custom_json_handling'
+  const client = createClient({
+    json: {
+      parse: JSON.parse,
+      stringify: (obj: unknown) => JSON.stringify(valueSerializer(obj)),
+    },
+  })
+  await client.command({
+    query: `DROP TABLE IF EXISTS ${tableName}`,
+  })
+  await client.command({
+    query: `
+      CREATE TABLE ${tableName}
+      (id UInt64, dt DateTime64(3, 'UTC'))
+      ENGINE MergeTree()
+      ORDER BY (id)
+    `,
+  })
+  await client.insert({
+    table: tableName,
+    values: [
+      {
+        id: BigInt(250000000000000200),
+        dt: new Date(),
+      },
+    ],
+    format: 'JSONEachRow',
+  })
+  const rows = await client.query({
+    query: `SELECT * FROM ${tableName}`,
+    format: 'JSONEachRow',
+  })
+  console.info(await rows.json())
+  await client.close()
+})()

packages/client-common/__tests__/integration/data_types.test.ts

@@ -216,6 +216,58 @@ describe('data types', () => {
     })
   })
 
+  it('should work with custom JSON handling (BigInt and Date)', async () => {
+    const TEST_BIGINT = BigInt(25000000000000000)
+    const TEST_DATE = new Date('2023-12-06T10:54:48.123Z')
+    const values = [
+      {
+        big_id: TEST_BIGINT,
+        dt: TEST_DATE,
+      },
+    ]
+
+    const valueSerializer = (value: unknown): unknown => {
+      if (value instanceof Date) {
+        return value.getTime()
+      }
+      if (typeof value === 'bigint') {
+        return value.toString()
+      }
+      if (Array.isArray(value)) {
+        return value.map(valueSerializer)
+      }
+      if (value && typeof value === 'object') {
+        return Object.fromEntries(
+          Object.entries(value).map(([k, v]) => [k, valueSerializer(v)]),
+        )
+      }
+      return value
+    }
+
+    // modify the client to handle BigInt and Date serialization
+    client = createTestClient({
+      json: {
+        parse: JSON.parse,
+        stringify: (obj: unknown) => {
+          const seralized = valueSerializer(obj)
+          return JSON.stringify(seralized)
+        },
+      },
+    })
+
+    const table = await createTableWithFields(
+      client,
+      "big_id UInt64, dt DateTime64(3, 'UTC')",
+    )
+
+    await insertAndAssert(table, values, {}, [
+      {
+        dt: TEST_DATE.toISOString().replace('T', ' ').replace('Z', ''), // clickhouse returns DateTime64 in UTC without timezone info
+        big_id: TEST_BIGINT.toString(), // clickhouse by default returns UInt64 as string to be safe
+      },
+    ])
+  })
+
   it('should work with string enums', async () => {
     const values = [
       { e1: 'Foo', e2: 'Qaz' },
@@ -753,8 +805,9 @@ describe('data types', () => {
     table: string,
     data: T[],
     clickhouse_settings: ClickHouseSettings = {},
+    expectedDataBack?: unknown[],
   ) {
     await insertData(table, data, clickhouse_settings)
-    await assertData(table, data, clickhouse_settings)
+    await assertData(table, expectedDataBack ?? data, clickhouse_settings)
   }
 })

packages/client-common/__tests__/unit/config.test.ts

@@ -376,6 +376,10 @@ describe('config', () => {
         keep_alive: { enabled: true },
         application_id: undefined,
         http_headers: {},
+        json: {
+          parse: JSON.parse,
+          stringify: JSON.stringify,
+        },
       })
     })
 
@@ -426,6 +430,10 @@ describe('config', () => {
         log_writer: jasmine.any(LogWriter),
         keep_alive: { enabled: false },
         application_id: 'my_app',
+        json: {
+          parse: JSON.parse,
+          stringify: JSON.stringify,
+        },
       })
     })
 
@@ -496,6 +504,10 @@ describe('config', () => {
         keep_alive: { enabled: true },
         application_id: undefined,
         http_headers: {},
+        json: {
+          parse: JSON.parse,
+          stringify: JSON.stringify,
+        },
       })
     })
   })

packages/client-common/src/client.ts

@@ -10,11 +10,16 @@ import type {
   WithClickHouseSummary,
   WithResponseHeaders,
 } from '@clickhouse/client-common'
-import { type DataFormat, DefaultLogger } from '@clickhouse/client-common'
+import {
+  type DataFormat,
+  defaultJSONHandling,
+  DefaultLogger,
+} from '@clickhouse/client-common'
 import type { InsertValues, NonEmptyArray } from './clickhouse_types'
 import type { ImplementationDetails, ValuesEncoder } from './config'
 import { getConnectionParams, prepareConfigWithURL } from './config'
 import type { ConnPingResult } from './connection'
+import type { JSONHandling } from './parse/json_handling'
 import type { BaseResultSet } from './result'
 
 export interface BaseQueryParams {
@@ -170,6 +175,7 @@ export class ClickHouseClient<Stream = unknown> {
   private readonly sessionId?: string
   private readonly role?: string | Array<string>
   private readonly logWriter: LogWriter
+  private readonly jsonHandling: JSONHandling
 
   constructor(
     config: BaseClickHouseClientConfigOptions & ImplementationDetails<Stream>,
@@ -192,7 +198,12 @@ export class ClickHouseClient<Stream = unknown> {
       this.connectionParams,
     )
     this.makeResultSet = config.impl.make_result_set
-    this.valuesEncoder = config.impl.values_encoder
+    this.jsonHandling = {
+      ...defaultJSONHandling,
+      ...config.json,
+    }
+
+    this.valuesEncoder = config.impl.values_encoder(this.jsonHandling)
   }
 
   /**
@@ -231,6 +242,7 @@ export class ClickHouseClient<Stream = unknown> {
         })
       },
       response_headers,
+      this.jsonHandling,
     )
   }
 

packages/client-common/src/config.ts

@@ -3,6 +3,7 @@ import type { Connection, ConnectionParams } from './connection'
 import type { DataFormat } from './data_formatter'
 import type { Logger } from './logger'
 import { ClickHouseLogLevel, LogWriter } from './logger'
+import { defaultJSONHandling, type JSONHandling } from './parse/json_handling'
 import type { BaseResultSet } from './result'
 import type { ClickHouseSettings } from './settings'
 
@@ -86,6 +87,12 @@ export interface BaseClickHouseClientConfigOptions {
      *  @default true */
     enabled?: boolean
   }
+  /**
+   * Custom parsing when handling with JSON objects
+   *
+   * Defaults to using standard `JSON.parse` and `JSON.stringify`
+   */
+  json?: Partial<JSONHandling>
 }
 
 export type MakeConnection<
@@ -102,8 +109,13 @@ export type MakeResultSet<Stream> = <
   query_id: string,
   log_error: (err: Error) => void,
   response_headers: ResponseHeaders,
+  jsonHandling: JSONHandling,
 ) => ResultSet
 
+export type MakeValuesEncoder<Stream> = (
+  jsonHandling: JSONHandling,
+) => ValuesEncoder<Stream>
+
 export interface ValuesEncoder<Stream> {
   validateInsertValues<T = unknown>(
     values: InsertValues<Stream, T>,
@@ -150,7 +162,7 @@ export interface ImplementationDetails<Stream> {
   impl: {
     make_connection: MakeConnection<Stream>
     make_result_set: MakeResultSet<Stream>
-    values_encoder: ValuesEncoder<Stream>
+    values_encoder: MakeValuesEncoder<Stream>
     handle_specific_url_params?: HandleImplSpecificURLParams
   }
 }
@@ -241,6 +253,10 @@ export function getConnectionParams(
     keep_alive: { enabled: config.keep_alive?.enabled ?? true },
     clickhouse_settings: config.clickhouse_settings ?? {},
     http_headers: config.http_headers ?? {},
+    json: {
+      ...defaultJSONHandling,
+      ...config.json,
+    },
   }
 }
 

packages/client-common/src/connection.ts

@@ -1,3 +1,4 @@
+import type { JSONHandling } from '.'
 import type {
   WithClickHouseSummary,
   WithResponseHeaders,
@@ -21,6 +22,7 @@ export interface ConnectionParams {
   application_id?: string
   http_headers?: Record<string, string>
   auth: ConnectionAuth
+  json?: JSONHandling
 }
 
 export interface CompressionSettings {

packages/client-common/src/data_formatter/formatter.ts

@@ -1,3 +1,5 @@
+import type { JSONHandling } from '../parse'
+
 export const StreamableJSONFormats = [
   'JSONEachRow',
   'JSONStringsEachRow',
@@ -112,9 +114,13 @@ export function validateStreamFormat(
  * @param format One of the supported JSON formats: https://clickhouse.com/docs/en/interfaces/formats/
  * @returns string
  */
-export function encodeJSON(value: any, format: DataFormat): string {
+export function encodeJSON(
+  value: any,
+  format: DataFormat,
+  stringifyFn: JSONHandling['stringify'],
+): string {
   if ((SupportedJSONFormats as readonly string[]).includes(format)) {
-    return JSON.stringify(value) + '\n'
+    return stringifyFn(value) + '\n'
   }
   throw new Error(
     `The client does not support JSON encoding in [${format}] format.`,

packages/client-common/src/index.ts

@@ -80,8 +80,13 @@ export type {
   ParsedColumnTuple,
   ParsedColumnMap,
   ParsedColumnType,
+  JSONHandling,
+} from './parse'
+export {
+  SimpleColumnTypes,
+  parseColumnType,
+  defaultJSONHandling,
 } from './parse'
-export { SimpleColumnTypes, parseColumnType } from './parse'
 
 /** For implementation usage only - should not be re-exported */
 export {

packages/client-common/src/parse/index.ts

@@ -1 +1,2 @@
 export * from './column_types'
+export * from './json_handling'