feat: connection validation and logging (#6233)

buntarb · web-flow · commit 6dc48f8dc804 · 2023-03-03T17:43:20.000+01:00
diff --git a/packages/cubejs-athena-driver/src/AthenaDriver.ts b/packages/cubejs-athena-driver/src/AthenaDriver.ts
@@ -87,9 +87,17 @@ export class AthenaDriver extends BaseDriver implements DriverInterface {
        * Max pool size value for the [cube]<-->[db] pool.
        */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {},
   ) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-base-driver/src/BaseDriver.ts b/packages/cubejs-base-driver/src/BaseDriver.ts
@@ -121,13 +121,21 @@ const DbTypeValueMatcher = {
  * Base driver class.
  */
 export abstract class BaseDriver implements DriverInterface {
+  private testConnectionTimeoutValue = 10000;
+
   protected logger: any;
 
   /**
    * Class constructor.
    */
-  public constructor(_options = {}) {
-    //
+  public constructor(_options: {
+    /**
+     * Time to wait for a response from a connection after validation
+     * request before determining it as not valid. Default - 10000 ms.
+     */
+    testConnectionTimeout?: number,
+  } = {}) {
+    this.testConnectionTimeoutValue = _options.testConnectionTimeout || 10000;
   }
 
   protected informationSchemaQuery() {
@@ -312,7 +320,7 @@ export abstract class BaseDriver implements DriverInterface {
   }
 
   public testConnectionTimeout() {
-    return 10000;
+    return this.testConnectionTimeoutValue;
   }
 
   public async downloadTable(table: string, _options: ExternalDriverCompatibilities): Promise<DownloadTableMemoryData | DownloadTableCSVData> {
diff --git a/packages/cubejs-bigquery-driver/src/BigQueryDriver.ts b/packages/cubejs-bigquery-driver/src/BigQueryDriver.ts
@@ -77,9 +77,17 @@ export class BigQueryDriver extends BaseDriver implements DriverInterface {
        * Max pool size value for the [cube]<-->[db] pool.
        */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {}
   ) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts b/packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts
@@ -77,11 +77,26 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
    */
   public constructor(
     config: ClickHouseDriverOptions & {
+      /**
+       * Data source name.
+       */
       dataSource?: string,
+
+      /**
+       * Max pool size value for the [cube]<-->[db] pool.
+       */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {},
   ) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts b/packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts
@@ -176,6 +176,12 @@ export class DatabricksDriver extends JDBCDriver {
        * Max pool size value for the [cube]<-->[db] pool.
        */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {},
   ) {
     const dataSource =
diff --git a/packages/cubejs-dremio-driver/driver/DremioDriver.js b/packages/cubejs-dremio-driver/driver/DremioDriver.js
@@ -40,7 +40,9 @@ class DremioDriver extends BaseDriver {
    * Class constructor.
    */
   constructor(config = {}) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-druid-driver/src/DruidDriver.ts b/packages/cubejs-druid-driver/src/DruidDriver.ts
@@ -40,11 +40,26 @@ export class DruidDriver extends BaseDriver {
    */
   public constructor(
     config: DruidDriverConfiguration & {
+      /**
+       * Data source name.
+       */
       dataSource?: string,
+
+      /**
+       * Max pool size value for the [cube]<-->[db] pool.
+       */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {}
   ) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-elasticsearch-driver/driver/ElasticSearchDriver.js b/packages/cubejs-elasticsearch-driver/driver/ElasticSearchDriver.js
@@ -28,7 +28,9 @@ class ElasticSearchDriver extends BaseDriver {
    * Class constructor.
    */
   constructor(config = {}) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-firebolt-driver/src/FireboltDriver.ts b/packages/cubejs-firebolt-driver/src/FireboltDriver.ts
@@ -60,8 +60,21 @@ export class FireboltDriver extends BaseDriver implements DriverInterface {
    */
   public constructor(
     config: Partial<FireboltDriverConfiguration> & {
+      /**
+       * Data source name.
+       */
       dataSource?: string,
+
+      /**
+       * Max pool size value for the [cube]<-->[db] pool.
+       */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {},
   ) {
     super(config);
diff --git a/packages/cubejs-hive-driver/driver/HiveDriver.js b/packages/cubejs-hive-driver/driver/HiveDriver.js
@@ -62,7 +62,9 @@ class HiveDriver extends BaseDriver {
    * Class constructor.
    */
   constructor(config = {}) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-jdbc-driver/src/JDBCDriver.ts b/packages/cubejs-jdbc-driver/src/JDBCDriver.ts
@@ -76,11 +76,26 @@ export class JDBCDriver extends BaseDriver {
 
   public constructor(
     config: Partial<JDBCDriverConfiguration> & {
+      /**
+       * Data source name.
+       */
       dataSource?: string,
+
+      /**
+       * Max pool size value for the [cube]<-->[db] pool.
+       */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {}
   ) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
@@ -130,13 +145,22 @@ export class JDBCDriver extends BaseDriver {
         new Promise((resolve) => {
           const isValid = promisify(connection.isValid.bind(connection));
           const timeout = setTimeout(() => {
+            if (this.logger) {
+              this.logger('Connection validation failed by timeout', {
+                testConnectionTimeout: this.testConnectionTimeout(),
+              });
+            }
             resolve(false);
           }, this.testConnectionTimeout());
           isValid(0).then((valid: boolean) => {
             clearTimeout(timeout);
+            if (!valid && this.logger) {
+              this.logger('Connection validation failed', {});
+            }
             resolve(valid);
-          }).catch(() => {
+          }).catch((e: { stack?: string }) => {
             clearTimeout(timeout);
+            this.databasePoolError(e);
             resolve(false);
           });
         })
diff --git a/packages/cubejs-ksql-driver/src/KsqlDriver.ts b/packages/cubejs-ksql-driver/src/KsqlDriver.ts
@@ -86,11 +86,26 @@ export class KsqlDriver extends BaseDriver implements DriverInterface {
    */
   public constructor(
     config: Partial<KsqlDriverOptions> & {
+      /**
+       * Data source name.
+       */
       dataSource?: string,
+
+      /**
+       * Max pool size value for the [cube]<-->[db] pool.
+       */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {}
   ) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-materialize-driver/src/MaterializeDriver.ts b/packages/cubejs-materialize-driver/src/MaterializeDriver.ts
@@ -44,8 +44,21 @@ export class MaterializeDriver extends PostgresDriver {
    */
   public constructor(
     options: PostgresDriverConfiguration & {
+      /**
+       * Data source name.
+       */
       dataSource?: string,
+
+      /**
+       * Max pool size value for the [cube]<-->[db] pool.
+       */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {},
   ) {
     super(options);
diff --git a/packages/cubejs-mongobi-driver/src/MongoBIDriver.ts b/packages/cubejs-mongobi-driver/src/MongoBIDriver.ts
@@ -44,11 +44,26 @@ export class MongoBIDriver extends BaseDriver implements DriverInterface {
    */
   public constructor(
     config: MongoBIDriverConfiguration & {
+      /**
+       * Data source name.
+       */
       dataSource?: string,
+
+      /**
+       * Max pool size value for the [cube]<-->[db] pool.
+       */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {}
   ) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-mssql-driver/driver/MSSqlDriver.js b/packages/cubejs-mssql-driver/driver/MSSqlDriver.js
@@ -41,7 +41,9 @@ class MSSqlDriver extends BaseDriver {
    * Class constructor.
    */
   constructor(config = {}) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-mysql-aurora-serverless-driver/driver/AuroraServerlessMySqlDriver.js b/packages/cubejs-mysql-aurora-serverless-driver/driver/AuroraServerlessMySqlDriver.js
@@ -33,7 +33,9 @@ class AuroraServerlessMySqlDriver extends BaseDriver {
    * Class constructor.
    */
   constructor(config = {}) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
     
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-mysql-driver/src/MySqlDriver.ts b/packages/cubejs-mysql-driver/src/MySqlDriver.ts
@@ -97,11 +97,26 @@ export class MySqlDriver extends BaseDriver implements DriverInterface {
    */
   public constructor(
     config: MySqlDriverConfiguration & {
+      /**
+       * Data source name.
+       */
       dataSource?: string,
+
+      /**
+       * Max pool size value for the [cube]<-->[db] pool.
+       */
       maxPoolSize?: number,
+
+      /**
+       * Time to wait for a response from a connection after validation
+       * request before determining it as not valid. Default - 10000 ms.
+       */
+      testConnectionTimeout?: number,
     } = {}
   ) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-oracle-driver/driver/OracleDriver.js b/packages/cubejs-oracle-driver/driver/OracleDriver.js
@@ -58,7 +58,9 @@ class OracleDriver extends BaseDriver {
    * Class constructor.
    */
   constructor(config = {}) {
-    super();
+    super({
+      testConnectionTimeout: config.testConnectionTimeout,
+    });
 
     const dataSource =
       config.dataSource ||
diff --git a/packages/cubejs-postgres-driver/src/PostgresDriver.ts b/packages/cubejs-postgres-driver/src/PostgresDriver.ts
diff --git a/packages/cubejs-questdb-driver/src/QuestDriver.ts b/packages/cubejs-questdb-driver/src/QuestDriver.ts
diff --git a/packages/cubejs-redshift-driver/src/RedshiftDriver.ts b/packages/cubejs-redshift-driver/src/RedshiftDriver.ts
diff --git a/packages/cubejs-server-core/src/core/optionsValidate.ts b/packages/cubejs-server-core/src/core/optionsValidate.ts
diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts
diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts
diff --git a/packages/cubejs-server-core/test/unit/OptsHandler.test.ts b/packages/cubejs-server-core/test/unit/OptsHandler.test.ts
diff --git a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts
diff --git a/packages/cubejs-sqlite-driver/driver/SqliteDriver.js b/packages/cubejs-sqlite-driver/driver/SqliteDriver.js
