Feat: (De)serialization of bigint and long

Author: Georg Traar

.github/workflows/publish.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 22
           registry-url: https://registry.npmjs.org/
 
       - name: Install dependencies

.github/workflows/test.yml

@@ -12,14 +12,18 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        node-version: ['22.x', '23.x']
+
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Node.js
+      - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: ${{ matrix.node-version }}
 
       - name: Install dependencies
         run: npm install

README.md

@@ -5,13 +5,11 @@
 
 This library is a lightweight Node.js client derived from `node-crate` for interacting with CrateDB via its **HTTP endpoint**. Unlike libraries such as `node-postgres`, which use the PostgreSQL wire protocol, this client communicates with CrateDB's native HTTP API.
 
-> [!CAUTION]
-> **This library is primarily a proof of concept.**  
-> While it provides basic functionality to interact with CrateDB, it is not production-ready and lacks the robustness of established libraries.  
-> 
+> [!CAUTION] > **This library is primarily a proof of concept.**  
+> While it provides basic functionality to interact with CrateDB, it is not production-ready and lacks the robustness of established libraries.
+>
 > For production use, consider mature libraries like [`node-postgres`](https://node-postgres.com/) which leverage CrateDB's PostgreSQL compatibility. Use this client only for **testing, experimentation**, or if you know what you're doing. :wink:
 
-
 ## Installation
 
 To install `node-cratedb` using npm:
@@ -44,10 +42,10 @@ const client = new CrateDBClient({
   password: 'secretpassword!!',
   host: 'my.database-server.com',
   port: 4200,
-  ssl: true,             // Use HTTPS
-  keepAlive: true,       // Enable persistent connections
-  maxConnections: 20,         // Limit to 10 concurrent sockets
-  defaultSchema: 'my_schema' // Default schema for queries
+  ssl: true, // Use HTTPS
+  keepAlive: true, // Enable persistent connections
+  maxConnections: 20, // Limit to 10 concurrent sockets
+  defaultSchema: 'my_schema', // Default schema for queries
 });
 ```
 
@@ -58,29 +56,28 @@ import { CrateDBClient } from '@proddata/node-cratedb';
 
 const client = new CrateDBClient({
   host: 'my.database-server.com',
-  jwt: 'your.jwt.token.here',  // Use JWT for Bearer authentication
-  ssl: true
+  jwt: 'your.jwt.token.here', // Use JWT for Bearer authentication
+  ssl: true,
 });
 ```
 
-
 ### Configuration
 
 The `CrateDBClient` can be configured with either environment variables or directly with an options object. Below are the configuration options, along with their default values.
 
 #### Configuration Options
 
-| Option             | Type                | Default Value                                   | Description                                                     |
-|--------------------|---------------------|-------------------------------------------------|-----------------------------------------------------------------|
-| `user`             | `string`            | `'crate'` or `process.env.CRATEDB_USER`         | Database user.                                                  |
-| `password`         | `string` or `null`  | `''` or `process.env.CRATEDB_PASSWORD`          | Database password.                                              |
-| `host`             | `string`            | `'localhost'`  or `process.env.CRATEDB_HOST`    | Database host.                                                  |
-| `port`             | `number`            | `4200` or `process.env.CRATEDB_PORT`            | Database port.                                                  |
-| `defaultSchema`    | `string`            | `null` or `process.env.CRATEDB_DEFAULT_SCHEMA`  | Default schema for queries.                                     |
-| `connectionString` | `string`            | `null`                                          | Connection string, e.g., `https://user:password@host:port/`.    |
-| `ssl`              | `object` or `null`  | `null`                                          | SSL configuration;                                              |
-| `keepAlive`        | `boolean`           | `true`                                          | Enables HTTP keep-alive for persistent connections.             |
-| `maxConnections`   | `number`            | `20`                                      | Limits the maximum number of concurrent connections.            |
+| Option             | Type               | Default Value                                  | Description                                                  |
+| ------------------ | ------------------ | ---------------------------------------------- | ------------------------------------------------------------ |
+| `user`             | `string`           | `'crate'` or `process.env.CRATEDB_USER`        | Database user.                                               |
+| `password`         | `string` or `null` | `''` or `process.env.CRATEDB_PASSWORD`         | Database password.                                           |
+| `host`             | `string`           | `'localhost'` or `process.env.CRATEDB_HOST`    | Database host.                                               |
+| `port`             | `number`           | `4200` or `process.env.CRATEDB_PORT`           | Database port.                                               |
+| `defaultSchema`    | `string`           | `null` or `process.env.CRATEDB_DEFAULT_SCHEMA` | Default schema for queries.                                  |
+| `connectionString` | `string`           | `null`                                         | Connection string, e.g., `https://user:password@host:port/`. |
+| `ssl`              | `object` or `null` | `null`                                         | SSL configuration;                                           |
+| `keepAlive`        | `boolean`          | `true`                                         | Enables HTTP keep-alive for persistent connections.          |
+| `maxConnections`   | `number`           | `20`                                           | Limits the maximum number of concurrent connections.         |
 
 #### Environment Variables
 
@@ -114,10 +111,9 @@ await client.execute('SELECT ?;', ['Hello World!']);
 Execute a raw bulk SQL query.
 
 ```js
-await client.execute('SELECT ?;', [['Hello'],['World']]);
+await client.execute('SELECT ?;', [['Hello'], ['World']]);
 ```
 
-
 #### streamQuery(sql, batchSize)
 
 The `streamQuery` method in CrateDBClient wraps the Cursor functionality
@@ -147,10 +143,10 @@ If `primaryKeys` are provided, the method will handle conflicts by updating the
 
 ```javascript
 // Insert a row with primary key conflict resolution
-await client.insert("my_table", { id: 1, column1: "value1", column2: "value2" }, ["id"]);
+await client.insert('my_table', { id: 1, column1: 'value1', column2: 'value2' }, ['id']);
 
 // Insert a row without conflict resolution
-await client.insert("my_table", { id: 1, column1: "value1", column2: "value2" });
+await client.insert('my_table', { id: 1, column1: 'value1', column2: 'value2' });
 ```
 
 #### insertMany(tableName, jsonArray, primaryKeys = null)
@@ -165,15 +161,15 @@ If `primaryKeys` are provided, the method will handle conflicts by updating the
 
 ```javascript
 const bulkData = [
-  { id: 1, name: "Earth", kind: "Planet", description: "A beautiful place." },
-  { id: 2, name: "Mars", kind: "Planet", description: "The red planet." },
-  { id: 1, name: "Earth Updated", kind: "Planet", description: "Updated description." }, // Conflict on id
+  { id: 1, name: 'Earth', kind: 'Planet', description: 'A beautiful place.' },
+  { id: 2, name: 'Mars', kind: 'Planet', description: 'The red planet.' },
+  { id: 1, name: 'Earth Updated', kind: 'Planet', description: 'Updated description.' }, // Conflict on id
 ];
 
-await client.insertMany("my_table", bulkData, ["id"]);
+await client.insertMany('my_table', bulkData, ['id']);
 // Conflicting row with `id: 1` will be updated instead of skipped.
 
-await client.insertMany("my_table", bulkData);
+await client.insertMany('my_table', bulkData);
 // Conflicting rows will be skipped as no `primaryKeys` are provided.
 ```
 
@@ -209,7 +205,6 @@ Refresh a specified table.
 await client.refresh('my_table');
 ```
 
-
 #### createTable(schema)
 
 Create a new table based on a schema definition.
@@ -219,8 +214,8 @@ await client.createTable({
   my_table: {
     id: 'INTEGER PRIMARY KEY',
     name: 'STRING',
-    created_at: 'TIMESTAMP'
-  }
+    created_at: 'TIMESTAMP',
+  },
 });
 ```
 
@@ -257,8 +252,27 @@ for await (const row of cursor.iterate(5)) {
 await cursor.close();
 ```
 
+## Handling JavaScript `BigInt` and CrateDB `LONG` Values
+
+This library leverages modern JavaScript features — such as `JSON.rawJSON()` -
+to accurately serialize and deserialize `BigInt` values.
+
+### Serialization
+
+- **BigInt to LONG:**
+  When serializing, JavaScript `BigInt` values their precision is preserved.
+  e.g. `BigInt(12345678901234567890)` is serialized as `12345678901234567890`.
+
+### Deserialization
+
+- **Top-Level LONG Columns:**  
+  If type information is available in the result set, columns defined as CrateDB  
+  `LONG` are automatically converted to `BigInt`.
+
+- **Large Integer Values:**  
+  If type information is unavailable, integers exceeding `Number.MAX_SAFE_INTEGER`
+  and without a decimal point are converted to `BigInt` on a best-effort basis.
 
 ## License
 
 MIT License. Feel free to use and modify this library as per the terms of the license.
-

package.json

@@ -49,6 +49,6 @@
     "vitest": "^2.1.5"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=21.0.0"
   }
-}
+}

src/CrateDBClient.ts

@@ -3,6 +3,7 @@ import http, { AgentOptions } from 'http';
 import https from 'https';
 import { URL } from 'url';
 import { CrateDBCursor } from './CrateDBCursor.js';
+import { CrateDBSerializer } from '../src/CrateDBSerializer';
 import {
   CrateDBConfig,
   CrateDBBaseResponse,
@@ -120,7 +121,7 @@ export class CrateDBClient {
     bulk_args: unknown[][] | null = null
   ): Promise<CrateDBBaseResponse> {
     const startRequestTime = Date.now();
-    const body = JSON.stringify(args ? { stmt, args } : { stmt, bulk_args });
+    const body = CrateDBSerializer.stringify(args ? { stmt, args } : { stmt, bulk_args });
     const options = { ...this.httpOptions, body };
     const response = await this._makeRequest(options);
     const totalRequestTime = Date.now() - startRequestTime;
@@ -271,7 +272,7 @@ export class CrateDBClient {
           const rawResponse = Buffer.concat(data); // Raw response data as a buffer
           const responseBodySize = rawResponse.length;
           try {
-            const parsedResponse = JSON.parse(rawResponse.toString());
+            const parsedResponse = CrateDBSerializer.deserialize(rawResponse.toString());
             resolve({
               ...parsedResponse,
               sizes: { response: responseBodySize, request: requestBodySize },

src/CrateDBCursor.ts

@@ -4,6 +4,7 @@ import http from 'http';
 import https from 'https';
 import { CrateDBClient } from './CrateDBClient.js';
 import { CrateDBResponse, CrateDBRecord } from './interfaces';
+import { CrateDBSerializer } from './CrateDBSerializer.js';
 
 export class CrateDBCursor {
   public client: CrateDBClient;
@@ -86,7 +87,7 @@ export class CrateDBCursor {
   }
 
   async _execute(sql: string): Promise<Array<CrateDBRecord>> {
-    const options = { ...this.connectionOptions, body: JSON.stringify({ stmt: sql }) };
+    const options = { ...this.connectionOptions, body: CrateDBSerializer.stringify({ stmt: sql }) };
     try {
       const response: CrateDBResponse = await this.client._makeRequest(options);
       const { cols, rows, rowcount } = response;

src/CrateDBSerializer.ts

@@ -0,0 +1,53 @@
+import { CrateDBTypes } from './CrateDBTypes';
+import { CrateDBBaseResponse } from './interfaces';
+
+export class CrateDBSerializer {
+  static stringify(obj: unknown): string {
+    return JSON.stringify(obj, replacer);
+  }
+
+  static parse(str: string): CrateDBBaseResponse {
+    return JSON.parse(str, reviver);
+  }
+
+  static deserialize(str: string) {
+    const obj = this.parse(str);
+    obj.col_types?.forEach((type: number, index: number) => {
+      switch (type) {
+        case CrateDBTypes.BIGINT:
+          obj.rows?.forEach((row: Array<unknown>) => {
+            if (typeof row[index] === 'number') {
+              row[index] = BigInt(String(row[index]));
+            }
+          });
+          break;
+        default:
+      }
+    });
+    return obj;
+  }
+}
+
+function replacer(_: unknown, value: unknown) {
+  if (typeof value === 'bigint') {
+    return JSON.rawJSON(value);
+  }
+  return value;
+}
+
+type Context = {
+  source: string;
+};
+
+function reviver(_: unknown, value: unknown, context: Context | null = null): unknown {
+  //if number is greater than Number.MAX_SAFE_INTEGER and not a float
+  if (
+    typeof value === 'number' &&
+    value > Number.MAX_SAFE_INTEGER &&
+    context !== null &&
+    !context.source.includes('.')
+  ) {
+    return BigInt(context.source);
+  }
+  return value;
+}

src/CrateDBTypes.ts

@@ -0,0 +1,31 @@
+export enum CrateDBTypes {
+  NULL = 0,
+  NOT_SUPPORTED = 1,
+  CHAR = 2,
+  BOOLEAN = 3,
+  TEXT = 4,
+  IP = 5,
+  DOUBLE_PRECISION = 6,
+  REAL = 7,
+  SMALLINT = 8,
+  INTEGER = 9,
+  BIGINT = 10,
+  TIMESTAMP_WITH_TIME_ZONE = 11,
+  OBJECT = 12,
+  GEO_POINT = 13,
+  GEO_SHAPE = 14,
+  TIMESTAMP_WITHOUT_TIME_ZONE = 15,
+  UNCHECKED_OBJECT = 16,
+  INTERVAL = 17,
+  REGPROC = 19,
+  TIME = 20,
+  OIDVECTOR = 21,
+  NUMERIC = 22,
+  REGCLASS = 23,
+  DATE = 24,
+  BIT = 25,
+  JSON = 26,
+  CHARACTER = 27,
+  FLOAT_VECTOR = 28,
+  ARRAY = 100,
+}

src/interfaces.ts

@@ -13,6 +13,8 @@ export interface CrateDBConfig {
 
 export interface CrateDBBaseResponse {
   cols?: string[];
+  col_types?: number[];
+  rows?: Array<Array<unknown>>;
   duration?: number;
   durations: {
     cratedb?: number;
@@ -27,7 +29,6 @@ export interface CrateDBBaseResponse {
 }
 
 export interface CrateDBResponse extends CrateDBBaseResponse {
-  col_types?: number[];
   rows?: Array<Array<unknown>>;
   rowcount?: number;
 }

src/json-extensions.d.ts

@@ -0,0 +1,8 @@
+// Extend the JSON interface to include rawJSON, available in Node.js v21.0.0+.
+// More info: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/rawJSON
+declare global {
+  interface JSON {
+    rawJSON(value: unknown): unknown;
+  }
+}
+export {};