Merge branch 'master' into copilot/implement-protocol-16-support

Author: mariuz

README.md

@@ -113,9 +113,9 @@ pool.destroy();
 
 ### Database Methods
 
-- `db.query(query, [params], function(err, result))` - classic query, returns Array of Object
-- `db.execute(query, [params], function(err, result))` - classic query, returns Array of Array
-- `db.sequentially(query, [params], function(row, index), function(err))` - sequentially query
+- `db.query(query, [params], function(err, result), options)` - classic query, returns Array of Object
+- `db.execute(query, [params], function(err, result), options)` - classic query, returns Array of Array
+- `db.sequentially(query, [params], function(row, index), function(err), options)` - sequentially query
 - `db.detach(function(err))` detach a database
 - `db.transaction(options, function(err, transaction))` create transaction
 
@@ -135,11 +135,20 @@ const options = {
 
 ### Transaction methods
 
-- `transaction.query(query, [params], function(err, result))` - classic query, returns Array of Object
-- `transaction.execute(query, [params], function(err, result))` - classic query, returns Array of Array
+- `transaction.query(query, [params], function(err, result), options)` - classic query, returns Array of Object
+- `transaction.execute(query, [params], function(err, result), options)` - classic query, returns Array of Array
+- `transaction.sequentially(query, [params], function(row, index), function(err), options)` - sequentially query
 - `transaction.commit(function(err))` commit current transaction
 - `transaction.rollback(function(err))` rollback current transaction
 
+### Statement options 
+
+```js
+const options = {
+  timeout: 1000, // Statement timeout in ms, default is 0 (no timeout)
+}
+```
+
 ## Examples
 
 ### Parametrized Queries
@@ -683,6 +692,29 @@ Firebird.attach({
    - Working with DECFLOAT values as strings or Buffers to preserve exact precision
 
 For legacy Firebird 4 servers with SRP authentication only, use the following configuration in `firebird.conf`:
+Firebird 4 wire protocol (versions 16 and 17) is partially supported, including:
+- **Time Zone Support**: Native support for `TIME WITH TIME ZONE` and `TIMESTAMP WITH TIME ZONE` (Protocol 16+).
+- **INT128 support**: Native support for 128-bit integers.
+- **Statement Timeout**: Support for statement-level timeouts.
+
+#### Using Timezones (FB 4.0+)
+
+Columns of type `TIMESTAMP WITH TIME ZONE` and `TIME WITH TIME ZONE` are automatically mapped to JavaScript `Date` objects. Values are read as UTC and represented in the local timezone of the Node.js process.
+
+```js
+// Select timezone columns
+db.query('SELECT TS_TZ_COL, T_TZ_COL FROM FB4_TABLE', function(err, result) {
+    console.log(result[0].ts_tz_col); // JavaScript Date object
+});
+
+// Insert using Date objects
+db.query('INSERT INTO FB4_TABLE (TS_TZ_COL) VALUES (?)', [new Date()], function(err) {
+    // ...
+});
+```
+
+Srp256 authentication and wire encryption are now supported natively,
+so you only need the following minimal configuration in `firebird.conf`:
 
 ```bash
 AuthServer = Srp256, Srp

Roadmap.md

@@ -12,6 +12,8 @@ The following table summarizes the current and planned implementation status of
 | 3.0 | 14, 15 | ✅ Implemented |
 | 4.0 | 16, 17 | ✅ Implemented (Partial) |
 | 5.0 | N/A | ❌ Not Implemented |
+| 4.0 | 16, 17 | ✅ Protocol 16 Implemented / 🚧 Protocol 17 Missing |
+| 5.0 | 18 | ❌ Not Implemented |
 | 6.0 | N/A | ❌ Not Implemented |
 
 ## Firebird 3 Support
@@ -27,8 +29,6 @@ Firebird 3 introduced Protocol 13, which brought significant changes focusing on
 - **UTF-8 User Identification:** ✅ Implemented - all user identification is properly handled with UTF-8 encoding via `isc_dpb_utf8_filename` flag for Firebird 3+.
 - **Database Encryption Callback:** ✅ Implemented - support for database encryption key callback (`op_crypt_key_callback`) during the connect phase, allowing connections to encrypted databases. The `dbCryptConfig` connection option supports both plain text and base64-encoded encryption keys.
 
-The following features are planned for future implementation:
-
 ## Firebird 4 Support
 
 Firebird 4 introduced Protocol versions 16 and 17, continuing to build upon the foundation of Firebird 3. The following features have been implemented:
@@ -44,23 +44,33 @@ The following features are planned for future implementation:
 - **Statement Timeouts:** Implement support for setting maximum execution times for SQL statements at the wire protocol level (Protocol 16 feature).
 - **Time Zone Data Types:** Add support for TIMESTAMP WITH TIME ZONE and TIME WITH TIME ZONE types.
 - **Full IEEE 754 DECFLOAT Support:** Implement proper IEEE 754 Decimal64 and Decimal128 encoding/decoding for DECFLOAT types (currently uses simplified implementation).
+- **Protocol Versions 16 and 17:** ✅ Protocol 16 Implemented - support for statement timeout, INT128, and timezones. 🚧 Protocol 17 Missing.
+- **Statement Timeout:** ✅ Implemented - support for `op_execute` with statement timeout (Protocol 16+).
+- **`INT128` Data Type:** ✅ Implemented - support for 128-bit integer data type.
+- **Time Zone Support:** ✅ Implemented - support for `TIME WITH TIME ZONE`, `TIMESTAMP WITH TIME ZONE`, and `sessionTimeZone` connection option (Protocol 16+).
+- **`DECFLOAT` Data Type:** ❌ TODO - support for `DECFLOAT(16)` and `DECFLOAT(34)`.
 
 ## Firebird 5 Support
 
-Firebird 5 introduces a host of new SQL features and performance improvements that will require significant client-side implementation:
+Firebird 5 introduces Protocol version 18 and a host of new SQL features and performance improvements:
+
+- **Protocol Version 18:** ❌ TODO - implement the latest protocol version.
+- **Bidirectional Cursors:** ❌ TODO - implement support for scrollable cursors for remote database access.
+- **`RETURNING` Multiple Rows:** ❌ TODO - enhance DML operations to support returning multiple rows.
+- **`SKIP LOCKED`:** ❌ TODO - add support for the `SKIP LOCKED` clause in `SELECT WITH LOCK`, `UPDATE`, and `DELETE` statements.
+- **Parallel Workers Information:** ❌ TODO - support for parallel workers information in SQL information items.
 
-- **Bidirectional Cursors:** Implement support for scrollable cursors for remote database access.
-- **`RETURNING` Multiple Rows:** Enhance DML operations to support returning multiple rows.
-- **`SKIP LOCKED`:** Add support for the `SKIP LOCKED` clause in `SELECT WITH LOCK`, `UPDATE`, and `DELETE` statements.
-- **New Data Types and Functions:** Add support for new built-in functions and packages.
 
 ## Firebird 6 and Beyond
 
 As Firebird 6 and future versions are released, we will continue to monitor new features and plan for their implementation. Key areas of interest include:
 
-- **JSON Support:** Implement client-side support for the new SQL-compliant JSON functions.
-- **Tablespaces:** Add support for tablespaces.
-- **SQL Schemas:** Implement support for SQL schemas.
+- **Native `JSON` Data Type:** Implement support for the new native `JSON` type (often handled as optimized binary storage).
+- **SQL-Standard `ROW` Type:** Support for structured data types (records) as columns or variables.
+- **SQL-Compliant JSON Functions:** Implement client-side support for `JSON_VALUE`, `JSON_QUERY`, `JSON_EXISTS`, and `JSON_OBJECT`.
+- **Tablespaces:** Add support for tablespaces to control physical storage locations.
+- **SQL Schemas:** Implement support for SQL-standard schemas for better namespace organization.
+- **Enhanced Collation Support:** Support for collations defined directly as part of the data type declaration.
 
 ## Codebase Refactoring
 

lib/index.d.ts

@@ -7,7 +7,7 @@ declare module 'node-firebird' {
     type TransactionCallback = (err: any, transaction: Transaction) => void;
     type QueryCallback = (err: any, result: any[]) => void;
     type SimpleCallback = (err: any) => void;
-    type SequentialCallback = (row: any, index: number) => void;
+    type SequentialCallback = (row: any, index: number, next?: (err?: any) => void) => void | Promise<void>;
 
     export const AUTH_PLUGIN_LEGACY: string;
     export const AUTH_PLUGIN_SRP: string;
@@ -41,21 +41,25 @@ declare module 'node-firebird' {
         waitTimeout?: number;
     };
 
+    export type QueryOptions = {
+        timeout: number;
+    }
+
     export interface Database {
         detach(callback?: SimpleCallback): Database;
         transaction(options: TransactionOptions|Isolation|TransactionCallback, callback?: TransactionCallback): Database;
-        query(query: string, params: any[], callback: QueryCallback): Database;
-        execute(query: string, params: any[], callback: QueryCallback): Database;
-        sequentially(query: string, params: any[], rowCallback: SequentialCallback, callback: SimpleCallback, asArray?: boolean): Database;
+        query(query: string, params: any[], callback: QueryCallback, options?: QueryOptions): Database;
+        execute(query: string, params: any[], callback: QueryCallback, options?: QueryOptions): Database;
+        sequentially(query: string, params: any[], rowCallback: SequentialCallback, callback: SimpleCallback, options?: QueryOptions | boolean): Database;
         drop(callback: SimpleCallback): void;
         escape(value: any): string;
         attachEvent(callback: any): this;
     }
 
     export interface Transaction {
-        query(query: string, params: any[], callback: QueryCallback): void;
-        execute(query: string, params: any[], callback: QueryCallback): void;
-        sequentially(query: string, params: any[], rowCallback: SequentialCallback, callback: SimpleCallback, asArray?: boolean): Database;
+        query(query: string, params: any[], callback: QueryCallback, options?: QueryOptions): void;
+        execute(query: string, params: any[], callback: QueryCallback, options?: QueryOptions): void;
+        sequentially(query: string, params: any[], rowCallback: SequentialCallback, callback: SimpleCallback, options?: QueryOptions | boolean): Database;
         commit(callback?: SimpleCallback): void;
         commitRetaining(callback?: SimpleCallback): void;
         rollback(callback?: SimpleCallback): void;

lib/wire/connection.js

@@ -408,6 +408,10 @@ class Connection {
         if (this.accept.authData) {
             blr.addString(Const.isc_dpb_specific_auth_data, this.accept.authData, Const.DEFAULT_ENCODING);
         }
+
+        if (options.sessionTimeZone) {
+            blr.addString(Const.isc_dpb_session_time_zone, options.sessionTimeZone, Const.DEFAULT_ENCODING);
+        }
 
         msg.addInt(Const.op_attach);
         msg.addInt(0);  // Database Object ID
@@ -508,6 +512,10 @@ class Connection {
         if (this.accept.authData) {
             blr.addString(Const.isc_dpb_specific_auth_data, this.accept.authData, Const.DEFAULT_ENCODING);
         }
+
+        if (options.sessionTimeZone) {
+            blr.addString(Const.isc_dpb_session_time_zone, options.sessionTimeZone, Const.DEFAULT_ENCODING);
+        }
 
         blr.addNumeric(Const.isc_dpb_sql_dialect, 3);
         blr.addNumeric(Const.isc_dpb_force_write, 1);
@@ -1011,6 +1019,10 @@ class Connection {
                         case Const.SQL_TYPE_DATE:
                         case Const.SQL_TYPE_TIME:
                         case Const.SQL_TIMESTAMP:
+                        case Const.SQL_TIME_TZ:
+                        case Const.SQL_TIME_TZ_EX:
+                        case Const.SQL_TIMESTAMP_TZ:
+                        case Const.SQL_TIMESTAMP_TZ_EX:
                             ret[i] = new Xsql.SQLParamDate(null);
                             break;
                         case Const.SQL_BLOB:
@@ -1034,12 +1046,15 @@ class Connection {
                             putBlobData(i, value, done);
                             break;
 
-                        case Const.SQL_TIMESTAMP:
-                        case Const.SQL_TYPE_DATE:
-                        case Const.SQL_TYPE_TIME:
-    
-                            if (value instanceof Date)
-                                ret[i] = new Xsql.SQLParamDate(value);
+                                                case Const.SQL_TIMESTAMP:
+                                                case Const.SQL_TYPE_DATE:
+                                                case Const.SQL_TYPE_TIME:
+                                                case Const.SQL_TIME_TZ:
+                                                case Const.SQL_TIME_TZ_EX:
+                                                case Const.SQL_TIMESTAMP_TZ:
+                                                case Const.SQL_TIMESTAMP_TZ_EX:
+                        
+                                                    if (value instanceof Date)                                ret[i] = new Xsql.SQLParamDate(value);
                             else if (typeof(value) === 'string')
                                 ret[i] = new Xsql.SQLParamDate(parseDate(value));
                             else
@@ -1173,6 +1188,11 @@ class Connection {
             }
             msg.addInt(0); // out_message_number = out_message_type
         }
+
+        if (this.accept.protocolVersion >= Const.PROTOCOL_VERSION16) {
+            // TODO impl statement timout
+            msg.addInt(statement.options?.timeout || 0); // p_sqldata_timeout
+        }
 
         callback.statement = statement;
         this._queueEvent(callback);
@@ -1207,41 +1227,56 @@ class Connection {
 
 
     fetchAll(statement, transaction, callback) {
-        const self = this, data = [];
+        const self = this;
+        const custom = statement.options || {};
+        const asStream = custom.asStream && custom.on;
+        const data = asStream ? null : [];
+        let streamIndex = 0;
         const loop = (err, ret) => {
             if (err) {
-                return callback(err);
-            } else if (ret && ret.data && ret.data.length) {
-                const arrPromise = (ret.arrBlob || []).map(value => value(transaction));
-    
+                callback(err);
+                return;
+            }
+
+            if (ret && ret.data && ret.data.length) {
+                const arrPromise = (ret.arrBlob || []).map((value) => value(transaction));
+
                 Promise.all(arrPromise).then((arrBlob) => {
                     for (let i = 0; i < arrBlob.length; i++) {
                         const blob = arrBlob[i];
                         ret.data[blob.row][blob.column] = blob.value;
                     }
-    
-                    const lastIndex = ret.data.length - 1;
-                    for (let i = 0; i < ret.data.length; i++) {
-                        const pos = data.push(ret.data[i]);
-                        if (statement.custom && statement.custom.asStream && statement.custom.on) {
-                            statement.custom.on(ret.data[i], pos - 1);
+
+                    doSynchronousLoop(ret.data, (row, _i, next) => {
+                        const pos = asStream ? streamIndex++ : (data.push(row) - 1);
+                        if (asStream) {
+                            executeStreamRow(custom, row, pos, statement.output, next);
+                        } else {
+                            next();
                         }
-                        if (i === lastIndex) {
-                            if (ret.fetched) {
-                                return callback(undefined, data);
-                            } else {
-                                self.fetch(statement, transaction, Const.DEFAULT_FETCHSIZE, loop);
-                            }
+                    }, (streamErr) => {
+                        if (streamErr) {
+                            callback(streamErr);
+                            return;
                         }
-                    }
+
+                        if (ret.fetched) {
+                            callback(undefined, data || []);
+                        } else {
+                            self.fetch(statement, transaction, Const.DEFAULT_FETCHSIZE, loop);
+                        }
+                    });
                 }).catch(callback);
-            } else if (ret.fetched) {
-                callback(undefined, data);
+                return;
+            }
+
+            if (ret && ret.fetched) {
+                callback(undefined, data || []);
             } else {
                 self.fetch(statement, transaction, Const.DEFAULT_FETCHSIZE, loop);
             }
-        }
-    
+        };
+
         this.fetch(statement, transaction, Const.DEFAULT_FETCHSIZE, loop);
     }
 
@@ -1560,7 +1595,7 @@ function decodeResponse(data, callback, cnx, lowercase_keys, cb) {
             case Const.op_sql_response:
                 var statement = callback.statement;
                 var output = statement.output;
-                var custom = statement.custom || {};
+                var custom = statement.options || {};
                 var isOpFetch = r === Const.op_fetch_response;
                 var _xdrpos;
                 statement.nbrowsfetched = statement.nbrowsfetched || 0;
@@ -1965,6 +2000,10 @@ function describe(buff, statement) {
                                 case Const.SQL_TYPE_DATE: param = new Xsql.SQLVarDate(); break;
                                 case Const.SQL_TYPE_TIME: param = new Xsql.SQLVarTime(); break;
                                 case Const.SQL_TIMESTAMP: param = new Xsql.SQLVarTimeStamp(); break;
+                                case Const.SQL_TIME_TZ:   param = new Xsql.SQLVarTimeTz(); break;
+                                case Const.SQL_TIME_TZ_EX: param = new Xsql.SQLVarTimeTzEx(); break;
+                                case Const.SQL_TIMESTAMP_TZ: param = new Xsql.SQLVarTimeStampTz(); break;
+                                case Const.SQL_TIMESTAMP_TZ_EX: param = new Xsql.SQLVarTimeStampTzEx(); break;
                                 case Const.SQL_BLOB:      param = new Xsql.SQLVarBlob(); break;
                                 case Const.SQL_ARRAY:     param = new Xsql.SQLVarArray(); break;
                                 case Const.SQL_QUAD:      param = new Xsql.SQLVarQuad(); break;
@@ -2191,6 +2230,57 @@ function fetch_blob_async(statement, id, name, row) {
     };
 }
 
+function doSynchronousLoop(data, processData, done) {
+    if (!data || !data.length) {
+        done();
+        return;
+    }
+
+    const loop = (index) => {
+        processData(data[index], index, (err) => {
+            if (err) {
+                done(err);
+                return;
+            }
+
+            const nextIndex = index + 1;
+            if (nextIndex < data.length) {
+                loop(nextIndex);
+            } else {
+                done();
+            }
+        });
+    };
+
+    loop(0);
+}
+
+function executeStreamRow(custom, row, index, output, next) {
+    let done = false;
+    const finish = (err) => {
+        if (done) {
+            return;
+        }
+        done = true;
+        next(err);
+    };
+
+    try {
+        const ret = custom.on(row, index, output, finish);
+        if (custom.on.length < 4) {
+            if (ret && typeof ret.then === 'function') {
+                ret.then(() => finish()).catch(finish);
+            } else {
+                finish();
+            }
+        } else if (ret && typeof ret.then === 'function') {
+            ret.catch(finish);
+        }
+    } catch (err) {
+        finish(err);
+    }
+}
+
 function bufferReader(buffer, max, writer, cb, beg, end) {
 
     if (!beg)