Merge pull request #90 from malthe/bigint-improvements

Add "bigints" option to change bigint behavior (allow the use of 'number' instead)

Author: malthe

CHANGES.md

@@ -1,5 +1,11 @@
 In next release ...
 
+- Both the `Query` object and client configuration now provide an optional `bigints`
+  setting which decides whether to use bigints or the number type for the INT8
+  data type. The setting is enabled by default.
+
+- Add support for the INT2 and INT8 array types.
+
 - The `Connect` and `End` events have been removed, in addition to the `Parameter`
   event; the `connect` method now returns an object with information about the
   established connection, namely whether the connection is encrypted and the

README.md

@@ -126,7 +126,7 @@ accessed via the iterator or record interface (see below).
 
 Query parameters use the format `$1`, `$2` etc.
 
-When a specific data type is not inferrable from the query, PostgreSQL
+When a specific data type can't be inferred from the query, PostgreSQL
 uses `DataType.Text` as the default data type (which is mapped to the
 string type in TypeScript). An explicit type can be provided in two
 different ways:
@@ -143,9 +143,19 @@ different ways:
     ```
 
 Note that the `number` type in TypeScript has a maximum safe integer
-value which lies between and `DataType.Int8` – given by
-`Number.MAX_SAFE_INTEGER`. To use `DataType.Int8` the `bigint` type
-should be used.
+value which is 2⁵³ – 1 (also given in the `Number.MAX_SAFE_INTEGER` constant),
+a value which lies between `DataType.Int4` and `DataType.Int8`. For numbers
+which can take on a value that's outside the safe range, use `DataType.Int8`
+(which translates to a `bigint` in TypeScript.)
+
+There's an optional setting `bigints` which can be configured on the client and/or
+specified for each query. It defaults to _true_, but can be set to _false_ in which
+case `number` is always used instead of `bigint` for `DataType.Int8` (throwing an 
+error if a query returns a value outside of the safe integer range.)
+
+Using a [check constraint](https://www.postgresql.org/docs/current/ddl-constraints.html)
+is recommended to ensure that values fit into the safe
+integer range, e.g. `CHECK (id < POWER(2, 53) - 1)`.
 
 ### Iterator interface
 

src/client.ts

@@ -75,6 +75,7 @@ export interface Configuration extends Partial<ClientConnectionDefaults & Client
     port?: number,
     password?: string,
     types?: Map<DataType, ValueTypeReader>,
+    bigints?: boolean,
     keepAlive?: boolean,
     preparedStatementPrefix?: string,
     connectionTimeout?: number,
@@ -117,6 +118,7 @@ type CloseHandler = () => void;
 interface RowDataHandler {
     callback: DataHandler,
     streams: Record<string, Writable>,
+    bigints: boolean,
 }
 
 type DescriptionHandler = (description: RowDescription) => void;
@@ -562,6 +564,7 @@ export class Client {
                                     handler: {
                                         callback: result.dataHandler,
                                         streams: streams || {},
+                                        bigints: query.bigints ?? this.config.bigints ?? true,
                                     },
                                     description: description,
                                 };
@@ -607,7 +610,7 @@ export class Client {
 
         const format = query?.format;
         const types = query?.types;
-        const streams =query?.streams;
+        const streams = query?.streams;
         const portal = query?.portal || '';
         const result = makeResult<T>(query?.transform);
 
@@ -618,6 +621,7 @@ export class Client {
         const dataHandler: RowDataHandler = {
             callback: result.dataHandler,
             streams: streams || {},
+            bigints: query.bigints ?? this.config.bigints ?? true,
         };
 
         if (values && values.length) {
@@ -815,7 +819,6 @@ export class Client {
     }
 
     private handle(buffer: Buffer, offset: number, size: number): number {
-        const types = this.config.types || null;
         let read = 0;
 
         while (size >= this.expect + read) {
@@ -825,6 +828,7 @@ export class Client {
             // Fast path: retrieve data rows.
             if (mtype === Message.RowData) {
                 const info = this.activeDataHandlerInfo;
+
                 if (!info) {
                     throw new Error('No active data handler');
                 }
@@ -835,8 +839,9 @@ export class Client {
 
                 const {
                     handler: {
-                        streams,
                         callback,
+                        streams,
+                        bigints,
                     },
                     description: {
                         columns,
@@ -846,16 +851,18 @@ export class Client {
 
                 let row = this.activeRow;
 
+                const types = this.config.types;
+                const encoding = this.encoding;
+
                 const hasStreams = Object.keys(streams).length > 0;
                 const mappedStreams = hasStreams ? names.map(
-                    name => streams[name] || null
-                ) : null;
+                    name => streams[name]
+                ) : undefined;
 
                 while (true) {
                     mtype = buffer.readInt8(frame);
                     if (mtype !== Message.RowData) break;
 
-
                     const bytes = buffer.readInt32BE(frame + 1) + 1;
                     const start = frame + 5;
 
@@ -875,7 +882,8 @@ export class Client {
                     const end = reader.readRowData(
                         row,
                         columns,
-                        this.encoding,
+                        encoding,
+                        bigints,
                         types,
                         mappedStreams
                     );

src/protocol.ts

@@ -285,8 +285,9 @@ export function readRowData(
     row: Array<any>,
     columnSpecification: Uint32Array,
     encoding: BufferEncoding,
-    types: ReadonlyMap<DataType, ValueTypeReader> | null,
-    streams: ReadonlyArray<Writable | null> | null,
+    bigints: boolean,
+    types?: ReadonlyMap<DataType, ValueTypeReader>,
+    streams?: ReadonlyArray<Writable | null>,
 ): number {
     const columns = row.length;
     const bufferLength = buffer.length;
@@ -319,9 +320,9 @@ export function readRowData(
             const spec = columnSpecification[j];
             let skip = false;
 
-            if (streams !== null && spec === DataType.Bytea) {
+            if (streams && spec === DataType.Bytea) {
                 const stream = streams[j];
-                if (stream !== null) {
+                if (stream) {
                     const slice = buffer.slice(start, end);
                     const alloc = Buffer.allocUnsafe(slice.length);
                     slice.copy(alloc, 0, 0, slice.length);
@@ -350,7 +351,7 @@ export function readRowData(
                 const isReader = (spec & readerMask) !== 0;
 
                 if (isReader) {
-                    const reader = (types) ? types.get(dataType) : null;
+                    const reader = types?.get(dataType);
                     if (reader) {
                         value = reader(
                             buffer,
@@ -404,8 +405,14 @@ export function readRowData(
                             case DataType.Int4:
                             case DataType.Oid:
                                 return buffer.readInt32BE(start);
-                            case DataType.Int8:
-                                return buffer.readBigInt64BE(start);
+                            case DataType.Int8: {
+                                const value = buffer.readBigInt64BE(start);
+                                if (bigints) return value;
+                                if (value > Number.MAX_SAFE_INTEGER) {
+                                    throw new Error("INT8 value too big for 'number' type");
+                                }
+                                return Number(value);
+                            }
                             case DataType.Float4:
                                 return buffer.readFloatBE(start);
                             case DataType.Float8:
@@ -631,14 +638,16 @@ export class Reader {
         row: any[],
         columnSpecification: Uint32Array,
         encoding: BufferEncoding,
-        types: ReadonlyMap<DataType, ValueTypeReader> | null,
-        streams: ReadonlyArray<Writable | null> | null,
+        bigints: boolean,
+        types?: ReadonlyMap<DataType, ValueTypeReader>,
+        streams?: ReadonlyArray<Writable | null>,
     ) {
         return readRowData(
             this.buffer.slice(this.start, this.end),
             row,
             columnSpecification,
             encoding,
+            bigints,
             types,
             streams
         );

src/query.ts

@@ -17,6 +17,8 @@ export interface QueryOptions {
     readonly streams: Record<string, Writable>;
     /** Allows the transformation of column names as returned by the database. */
     readonly transform: (name: string) => string;
+    /** Use bigint for the INT8 (64-bit integer) data type. */
+    readonly bigints: boolean; 
 }
 
 /**

src/types.ts

@@ -39,11 +39,13 @@ export enum DataType {
     Inet = 869,
     ArrayBytea = 1001,
     ArrayChar = 1002,
+    ArrayInt2 = 1005,
     ArrayInt4 = 1007,
     ArrayRegprocedure = 1008,
     ArrayText = 1009,
     ArrayBpchar = 1014,
     ArrayVarchar = 1015,
+    ArrayInt8 = 1016,
     ArrayFloat4 = 1021,
     ArrayFloat8 = 1022,
     Aclitem = 1033,
@@ -108,7 +110,9 @@ export const arrayDataTypeMapping: ReadonlyMap<DataType, DataType> = new Map([
     [DataType.ArrayDate, DataType.Date],
     [DataType.ArrayFloat4, DataType.Float4],
     [DataType.ArrayFloat8, DataType.Float8],
+    [DataType.ArrayInt2, DataType.Int2],
     [DataType.ArrayInt4, DataType.Int4],
+    [DataType.ArrayInt8, DataType.Int8],
     [DataType.ArrayJson, DataType.Json],
     [DataType.ArrayJsonb, DataType.Jsonb],
     [DataType.ArrayText, DataType.Text],

test/helper.ts

@@ -24,8 +24,8 @@ export function testWithClient(name: string, fn: Test, timeout?: number) {
             }
             if (!client.closed) {
                 await client.end();
-                if (!client.closed) throw new Error("Expected client to be closed");
             }
         }
+        if (!client.closed) throw new Error("Expected client to be closed");
     }, timeout);
 }

test/types.test.ts

@@ -28,17 +28,19 @@ function testType<T>(
     dataType: DataType,
     expression: string,
     expected: T,
-    excludeTextMode = false) {
+    excludeTextMode = false,
+    bigints?: boolean) {
     const testParam = (format: DataFormat) => {
         testWithClient('Param', async (client) => {
             expect.assertions(3);
-            const query = expected !== null
-                ? getComparisonQueryFor(dataType, expression)
+            const text = expected !== null
+                ? getComparisonQueryFor(dataType, expression) + ' where $1 is not null'
                 : 'select $1 is null';
             await client.query({
-                text: expected !== null ? query + ' where $1 is not null' : query,
+                text,
                 types: [dataType],
-                format
+                format,
+                bigints,
             }, [expected])
                 .then(
                     (result) => {
@@ -53,13 +55,13 @@ function testType<T>(
     const testValue = (format: DataFormat) => {
         testWithClient('Value', async (client) => {
             expect.assertions(3);
-            const query = 'select ' + expression;
-            await client.query({text: query, format}, []).then(
+            const text = 'select ' + expression;
+            await client.query({text, format, bigints}, []).then(
                 (result) => {
                     const rows = result.rows;
                     expect(rows.length).toEqual(1);
                     expect(rows[0].length).toEqual(1);
-                    expect(rows[0][0]).toEqual(expected)
+                    expect(rows[0][0]).toEqual(expected);
                 });
         })
     };
@@ -101,6 +103,7 @@ describe('Types', () => {
     testType<number>(DataType.Int2, '1::int2', 1);
     testType<number>(DataType.Int4, '1::int4', 1);
     testType<bigint>(DataType.Int8, '1::int8', BigInt(1));
+    testType<number>(DataType.Int8, '1::int8', 1, undefined, false);
     testType<number>(DataType.Float4, '1::float4', 1.0);
     testType<number>(DataType.Float8, '1::float8', 1.0);
     testType<number>(DataType.Oid, '1::oid', 1);
@@ -189,6 +192,10 @@ describe('Types', () => {
         'ARRAY[\'123e4567-e89b-12d3-a456-426655440000\'::uuid]',
         ['123e4567-e89b-12d3-a456-426655440000']
     );
+    testType<number[]>(
+        DataType.ArrayInt2,
+        '\'{1,2,3}\'::int2[3]',
+        [1, 2, 3]);
     testType<number[]>(
         DataType.ArrayInt4,
         '\'{1,2,3}\'::int4[3]',
@@ -201,6 +208,16 @@ describe('Types', () => {
         DataType.ArrayInt4,
         '\'{{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}}\'::int4[]',
         [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]);
+    testType<bigint[]>(
+        DataType.ArrayInt8,
+        '\'{1,2,3}\'::int8[3]',
+        [BigInt(1), BigInt(2), BigInt(3)]);
+    testType<number[]>(
+        DataType.ArrayInt8,
+        '\'{1,2,3}\'::int8[3]',
+        [1, 2, 3],
+        undefined,
+        false);
     testType<number[]>(
         DataType.ArrayFloat4,
         '\'{1.0, 2.0, 3.0}\'::float4[3]',