diff --git a/packages/cubejs-base-driver/src/BaseDriver.ts b/packages/cubejs-base-driver/src/BaseDriver.ts index fc2890b1ae4e7..d9b5123604c8d 100644 --- a/packages/cubejs-base-driver/src/BaseDriver.ts +++ b/packages/cubejs-base-driver/src/BaseDriver.ts @@ -408,7 +408,7 @@ export abstract class BaseDriver implements DriverInterface { }; } - public readOnly() { + public readOnly(): boolean { return false; } diff --git a/packages/cubejs-mssql-driver/.gitignore b/packages/cubejs-mssql-driver/.gitignore new file mode 100644 index 0000000000000..1521c8b7652b1 --- /dev/null +++ b/packages/cubejs-mssql-driver/.gitignore @@ -0,0 +1 @@ +dist diff --git a/packages/cubejs-mssql-driver/driver/QueryStream.js b/packages/cubejs-mssql-driver/driver/QueryStream.js deleted file mode 100644 index 239832982f8ba..0000000000000 --- a/packages/cubejs-mssql-driver/driver/QueryStream.js +++ /dev/null @@ -1,62 +0,0 @@ -const { Readable } = require('stream'); -const { getEnv } = require('@cubejs-backend/shared'); - -/** - * MS-SQL query stream class. - */ -class QueryStream extends Readable { - request = null; - toRead = 0; - - /** - * @constructor - */ - constructor(request, highWaterMark) { - super({ - objectMode: true, - highWaterMark: - highWaterMark || getEnv('dbQueryStreamHighWaterMark'), - }); - this.request = request; - this.request.on('row', row => { - this.transformRow(row); - const canAdd = this.push(row); - if (this.toRead-- <= 0 || !canAdd) { - this.request.pause(); - } - }) - this.request.on('done', () => { - this.push(null); - }) - this.request.on('error', (err) => { - this.destroy(err); - }); - } - - /** - * @override - */ - _read(toRead) { - this.toRead += toRead; - this.request.resume(); - } - - transformRow(row) { - for (const key in row) { - if (row.hasOwnProperty(key) && row[key] && row[key] instanceof Date) { - row[key] = row[key].toJSON(); - } - } - } - - /** - * @override - */ - _destroy(error, callback) { - this.request.cancel(); - this.request = null; - callback(error); - } -} - -module.exports = QueryStream; diff --git a/packages/cubejs-mssql-driver/driver/index.d.ts b/packages/cubejs-mssql-driver/driver/index.d.ts deleted file mode 100644 index e403e7dc6f9ae..0000000000000 --- a/packages/cubejs-mssql-driver/driver/index.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { BaseDriver } from "@cubejs-backend/query-orchestrator"; -import { config } from "mssql"; - -declare module "@cubejs-backend/mssql-driver" { - export default class MSSqlDriver extends BaseDriver { - constructor(options?: config); - release(): Promise - } -} diff --git a/packages/cubejs-mssql-driver/index.js b/packages/cubejs-mssql-driver/index.js new file mode 100644 index 0000000000000..b925f7fdcf294 --- /dev/null +++ b/packages/cubejs-mssql-driver/index.js @@ -0,0 +1,11 @@ +const fromExports = require('./dist/src'); +const { MSSqlDriver } = require('./dist/src/MSSqlDriver'); + +const toExport = MSSqlDriver; + +// eslint-disable-next-line no-restricted-syntax +for (const [key, module] of Object.entries(fromExports)) { + toExport[key] = module; +} + +module.exports = toExport; diff --git a/packages/cubejs-mssql-driver/package.json b/packages/cubejs-mssql-driver/package.json index 36769fd4f8a1a..e1fdd1af48a4d 100644 --- a/packages/cubejs-mssql-driver/package.json +++ b/packages/cubejs-mssql-driver/package.json @@ -11,18 +11,34 @@ "engines": { "node": "^14.0.0 || ^16.0.0 || >=17.0.0" }, - "main": "driver/MSSqlDriver.js", + "files": [ + "dist/src", + "index.js" + ], + "main": "index.js", + "typings": "dist/src/index.d.ts", + "scripts": { + "build": "rm -rf dist && npm run tsc", + "tsc": "tsc", + "watch": "tsc -w", + "lint": "eslint src/* --ext .ts,.js", + "lint:fix": "eslint --fix src/* --ext .ts,.js" + }, "dependencies": { "@cubejs-backend/base-driver": "1.3.5", - "mssql": "^10.0.2" + "@cubejs-backend/shared": "1.3.5", + "mssql": "^11.0.1" }, "devDependencies": { - "@types/mssql": "^9.1.5", + "@types/mssql": "^9.1.7", "@types/node": "^20" }, "jest": { "testEnvironment": "node" }, + "eslintConfig": { + "extends": "../cubejs-linter" + }, "license": "Apache-2.0", "publishConfig": { "access": "public" diff --git a/packages/cubejs-mssql-driver/driver/MSSqlDriver.js b/packages/cubejs-mssql-driver/src/MSSqlDriver.ts similarity index 65% rename from packages/cubejs-mssql-driver/driver/MSSqlDriver.js rename to packages/cubejs-mssql-driver/src/MSSqlDriver.ts index 00bd431b4c4e5..3688453395ab5 100644 --- a/packages/cubejs-mssql-driver/driver/MSSqlDriver.js +++ b/packages/cubejs-mssql-driver/src/MSSqlDriver.ts @@ -4,16 +4,46 @@ * @fileoverview The `MSSqlDriver` and related types declaration. */ -const { +import sql, { ConnectionPool, config as MsSQLConfig } from 'mssql'; +import { getEnv, assertDataSource, -} = require('@cubejs-backend/shared'); -const sql = require('mssql'); -const { BaseDriver } = require('@cubejs-backend/base-driver'); -const QueryStream = require('./QueryStream'); +} from '@cubejs-backend/shared'; +import { + BaseDriver, + DriverInterface, + StreamOptions, + DownloadQueryResultsOptions, + TableStructure, + DriverCapabilities, + DownloadQueryResultsResult, TableColumnQueryResult, +} from '@cubejs-backend/base-driver'; +import { QueryStream } from './QueryStream'; + +// ********* Value converters ***************** // +const numericTypes = [ + sql.TYPES.Int, + sql.TYPES.BigInt, + sql.TYPES.SmallInt, + sql.TYPES.TinyInt, + sql.TYPES.Decimal, + sql.TYPES.Numeric, + sql.TYPES.Float, + sql.TYPES.Real, + sql.TYPES.Money, + sql.TYPES.SmallMoney +]; + +for (const type of numericTypes) { + sql.valueHandler.set(type, (value) => (value != null ? String(value) : value)); +} +export type MSSqlDriverConfiguration = Omit & { + readOnly?: boolean; + server?: string; +}; -const GenericTypeToMSSql = { +const GenericTypeToMSSql: Record = { boolean: 'bit', string: 'nvarchar(max)', text: 'nvarchar(max)', @@ -21,27 +51,50 @@ const GenericTypeToMSSql = { uuid: 'uniqueidentifier' }; -const MSSqlToGenericType = { +const MSSqlToGenericType: Record = { bit: 'boolean', uniqueidentifier: 'uuid', datetime2: 'timestamp' -} +}; /** * MS SQL driver class. */ -class MSSqlDriver extends BaseDriver { +export class MSSqlDriver extends BaseDriver implements DriverInterface { + private readonly connectionPool: ConnectionPool; + + private readonly initialConnectPromise: Promise; + + private readonly config: MSSqlDriverConfiguration; + /** * Returns default concurrency value. */ - static getDefaultConcurrency() { + public static getDefaultConcurrency() { return 2; } /** * Class constructor. */ - constructor(config = {}) { + public constructor(config: MSSqlDriverConfiguration & { + /** + * Data source name. + */ + dataSource?: string, + + /** + * Max pool size value for the [cube]<-->[db] pool. + */ + maxPoolSize?: number, + + /** + * Time to wait for a response from a connection after validation + * request before determining it as not valid. Default - 10000 ms. + */ + testConnectionTimeout?: number, + server?: string, + } = {}) { super({ testConnectionTimeout: config.testConnectionTimeout, }); @@ -78,16 +131,16 @@ class MSSqlDriver extends BaseDriver { ...config }; const { readOnly, ...poolConfig } = this.config; - this.connectionPool = new sql.ConnectionPool(poolConfig); + this.connectionPool = new ConnectionPool(poolConfig as MsSQLConfig); this.initialConnectPromise = this.connectionPool.connect(); } /** * Returns the configurable driver options * Note: It returns the unprefixed option names. - * In case of using multisources options need to be prefixed manually. + * In case of using multi sources options need to be prefixed manually. */ - static driverEnvVariables() { + public static driverEnvVariables() { return [ 'CUBEJS_DB_HOST', 'CUBEJS_DB_NAME', @@ -98,8 +151,9 @@ class MSSqlDriver extends BaseDriver { ]; } - testConnection() { - return this.initialConnectPromise.then((pool) => pool.request().query('SELECT 1 as number')); + public async testConnection() { + const conn = await this.initialConnectPromise.then((pool: ConnectionPool) => pool.request()); + await conn.query('SELECT 1 as number'); } /** @@ -110,11 +164,7 @@ class MSSqlDriver extends BaseDriver { * @param {{ highWaterMark: number? }} options * @return {Promise} */ - async stream( - query, - values, - options, - ) { + public async stream(query: string, values: unknown[], { highWaterMark }: StreamOptions) { const pool = await this.initialConnectPromise; const request = pool.request(); @@ -124,17 +174,17 @@ class MSSqlDriver extends BaseDriver { }); request.query(query); - const stream = new QueryStream(request, options?.highWaterMark); - const fields = await new Promise((resolve, reject) => { + const stream = new QueryStream(request, highWaterMark); + const fields: TableStructure = await new Promise((resolve, reject) => { request.on('recordset', (columns) => { resolve(this.mapFields(columns)); }); - request.on('error', (err) => { + request.on('error', (err: Error) => { reject(err); }); - stream.on('error', (err) => { + stream.on('error', (err: Error) => { reject(err); - }) + }); }); return { rowStream: stream, @@ -161,7 +211,7 @@ class MSSqlDriver extends BaseDriver { * } * }} fields */ - mapFields(fields) { + private mapFields(fields: Record) { return Object.keys(fields).map((field) => { let type; switch (fields[field].type) { @@ -231,9 +281,9 @@ class MSSqlDriver extends BaseDriver { }); } - query(query, values) { - let cancelFn = null; - const promise = this.initialConnectPromise.then((pool) => { + public async query(query: string, values: unknown[]) { + let cancelFn: (() => void) | null = null; + const promise: any = this.initialConnectPromise.then((pool) => { const request = pool.request(); (values || []).forEach((v, i) => request.input(`_${i + 1}`, v)); @@ -246,14 +296,14 @@ class MSSqlDriver extends BaseDriver { return promise; } - param(paramIndex) { + public param(paramIndex: number): string { return `@_${paramIndex + 1}`; } - async tableColumnTypes(table) { + public async tableColumnTypes(table: string): Promise { const [schema, name] = table.split('.'); - const columns = await this.query( + const columns: TableColumnQueryResult[] = await this.query( `SELECT column_name as ${this.quoteIdentifier('column_name')}, table_name as ${this.quoteIdentifier('table_name')}, table_schema as ${this.quoteIdentifier('table_schema')}, @@ -266,27 +316,27 @@ class MSSqlDriver extends BaseDriver { return columns.map(c => ({ name: c.column_name, type: this.toGenericType(c.data_type) })); } - getTablesQuery(schemaName) { + public getTablesQuery(schemaName: string) { return this.query( `SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = ${this.param(0)}`, [schemaName] ); } - createSchemaIfNotExists(schemaName) { + public async createSchemaIfNotExists(schemaName: string): Promise { return this.query( `SELECT schema_name FROM INFORMATION_SCHEMA.SCHEMATA WHERE schema_name = ${this.param(0)}`, [schemaName] - ).then((schemas) => { + ).then((schemas: string[]) => { if (schemas.length === 0) { - return this.query(`CREATE SCHEMA ${schemaName}`); + return this.query(`CREATE SCHEMA ${schemaName}`, []); } return null; }); } - informationSchemaQuery() { - // fix The multi-part identifier "columns.data_type" could not be bound + public informationSchemaQuery(): string { + // fix The multipart identifier "columns.data_type" could not be bound return ` SELECT column_name as ${this.quoteIdentifier('column_name')}, table_name as ${this.quoteIdentifier('table_name')}, @@ -297,8 +347,8 @@ class MSSqlDriver extends BaseDriver { `; } - async downloadQueryResults(query, values, options) { - if ((options || {}).streamImport) { + public async downloadQueryResults(query: string, values: unknown[], options: DownloadQueryResultsOptions): Promise { + if (options?.streamImport) { return this.stream(query, values, options); } @@ -314,27 +364,25 @@ class MSSqlDriver extends BaseDriver { }; } - fromGenericType(columnType) { + protected fromGenericType(columnType: string): string { return GenericTypeToMSSql[columnType] || super.fromGenericType(columnType); } - toGenericType(columnType){ + protected toGenericType(columnType: string): string { return MSSqlToGenericType[columnType] || super.toGenericType(columnType); } - readOnly() { + public readOnly(): boolean { return !!this.config.readOnly; } - wrapQueryWithLimit(query) { + public wrapQueryWithLimit(query: { query: string, limit: number}) { query.query = `SELECT TOP ${query.limit} * FROM (${query.query}) AS t`; } - capabilities() { + public capabilities(): DriverCapabilities { return { incrementalSchemaLoading: true, }; } } - -module.exports = MSSqlDriver; diff --git a/packages/cubejs-mssql-driver/src/QueryStream.ts b/packages/cubejs-mssql-driver/src/QueryStream.ts new file mode 100644 index 0000000000000..1693c9a28c8b4 --- /dev/null +++ b/packages/cubejs-mssql-driver/src/QueryStream.ts @@ -0,0 +1,64 @@ +import { Readable } from 'stream'; +import sql from 'mssql'; +import { + getEnv, +} from '@cubejs-backend/shared'; + +/** + * MS-SQL query stream class. + */ +export class QueryStream extends Readable { + private request: sql.Request | null; + + private toRead: number = 0; + + /** + * @constructor + */ + public constructor(request: sql.Request, highWaterMark: number) { + super({ + objectMode: true, + highWaterMark: + highWaterMark || getEnv('dbQueryStreamHighWaterMark'), + }); + this.request = request; + this.request.on('row', row => { + this.transformRow(row); + const canAdd = this.push(row); + if (this.toRead-- <= 0 || !canAdd) { + this.request?.pause(); + } + }); + this.request.on('done', () => { + this.push(null); + }); + this.request.on('error', (err: Error) => { + this.destroy(err); + }); + } + + /** + * @override + */ + public _read(toRead: number) { + this.toRead += toRead; + this.request?.resume(); + } + + private transformRow(row: Record) { + for (const [key, value] of Object.entries(row)) { + if (value instanceof Date) { + row[key] = value.toJSON(); + } + } + } + + /** + * @override + */ + public _destroy(error: any, callback: CallableFunction) { + this.request?.cancel(); + this.request = null; + callback(error); + } +} diff --git a/packages/cubejs-mssql-driver/src/index.ts b/packages/cubejs-mssql-driver/src/index.ts new file mode 100644 index 0000000000000..fd696adf3c2c5 --- /dev/null +++ b/packages/cubejs-mssql-driver/src/index.ts @@ -0,0 +1,5 @@ +import { MSSqlDriver } from './MSSqlDriver'; + +export * from './MSSqlDriver'; + +export default MSSqlDriver; diff --git a/packages/cubejs-mssql-driver/src/types/mssql.d.ts b/packages/cubejs-mssql-driver/src/types/mssql.d.ts new file mode 100644 index 0000000000000..cecadcb197e4c --- /dev/null +++ b/packages/cubejs-mssql-driver/src/types/mssql.d.ts @@ -0,0 +1,13 @@ +import 'mssql'; + +// Because "@types/mssql": "^9.1.7" (latest as of Apr 2025) still doesn't have info about valueHandler +declare module 'mssql' { + namespace valueHandler { + function set( + type: any, + handler: (value: unknown) => unknown + ): void; + } + + export const valueHandler: typeof valueHandler; +} diff --git a/packages/cubejs-mssql-driver/tsconfig.json b/packages/cubejs-mssql-driver/tsconfig.json new file mode 100644 index 0000000000000..014fba0433cbb --- /dev/null +++ b/packages/cubejs-mssql-driver/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "include": [ + "src", + "test" + ], + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "baseUrl": ".", + "typeRoots": ["./node_modules/@types", "./src/types"] + } +} diff --git a/packages/cubejs-mysql-driver/package.json b/packages/cubejs-mysql-driver/package.json index 0cdda01999080..a0974bddbef34 100644 --- a/packages/cubejs-mysql-driver/package.json +++ b/packages/cubejs-mysql-driver/package.json @@ -29,7 +29,6 @@ "dependencies": { "@cubejs-backend/base-driver": "1.3.5", "@cubejs-backend/shared": "1.3.5", - "@types/mysql": "^2.15.21", "generic-pool": "^3.8.2", "mysql": "^2.18.1" }, @@ -37,6 +36,7 @@ "@cubejs-backend/linter": "1.3.5", "@cubejs-backend/testing-shared": "1.3.5", "@types/generic-pool": "^3.8.2", + "@types/mysql": "^2.15.21", "@types/jest": "^27", "jest": "^27", "stream-to-array": "^2.3.0", diff --git a/packages/cubejs-postgres-driver/src/PostgresDriver.ts b/packages/cubejs-postgres-driver/src/PostgresDriver.ts index e9a5a0dacb162..0a625bcf4e9a1 100644 --- a/packages/cubejs-postgres-driver/src/PostgresDriver.ts +++ b/packages/cubejs-postgres-driver/src/PostgresDriver.ts @@ -310,7 +310,7 @@ export class PostgresDriver=18", "@types/node@^18", "@types/node@^20", "@types/node@^22.5.5": version "20.17.28" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.17.28.tgz#c10436f3a3c996f535919a9b082e2c47f19c40a1" integrity sha512-DHlH/fNL6Mho38jTy7/JT7sn2wnXI+wULR6PV4gy4VHLVvnrV/d3pHAMQHhc4gjdLmK2ZiPoMxzp6B3yRajLSQ== @@ -10610,13 +10612,6 @@ dependencies: "@types/acorn" "*" -"@types/tedious@*": - version "4.0.14" - resolved "https://registry.yarnpkg.com/@types/tedious/-/tedious-4.0.14.tgz#868118e7a67808258c05158e9cad89ca58a2aec1" - integrity sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw== - dependencies: - "@types/node" "*" - "@types/throttle-debounce@^2.1.0": version "2.1.0" resolved "https://registry.yarnpkg.com/@types/throttle-debounce/-/throttle-debounce-2.1.0.tgz#1c3df624bfc4b62f992d3012b84c56d41eab3776" @@ -12707,10 +12702,10 @@ bl@^4.0.3, bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" -bl@^6.0.3: - version "6.0.12" - resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8" - integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w== +bl@^6.0.11, bl@^6.0.3: + version "6.1.0" + resolved "https://registry.yarnpkg.com/bl/-/bl-6.1.0.tgz#cc35ce7a2e8458caa8c8fb5deeed6537b73e4504" + integrity sha512-ClDyJGQkc8ZtzdAAbAwBmhMSpwN/sC9HA8jxdYm6nVUbCfZbe2mgza4qh7AuEYyEPB/c4Kznf9s66bnsKMQDjw== dependencies: "@types/readable-stream" "^4.0.0" buffer "^6.0.3" @@ -21255,6 +21250,18 @@ mssql@^10.0.2: tarn "^3.0.2" tedious "^16.4.0" +mssql@^11.0.1: + version "11.0.1" + resolved "https://registry.yarnpkg.com/mssql/-/mssql-11.0.1.tgz#a32ab7763bfbb3f5d970e47563df3911fc04e21d" + integrity sha512-KlGNsugoT90enKlR8/G36H0kTxPthDhmtNUCwEHvgRza5Cjpjoj+P2X6eMpFUDN7pFrJZsKadL4x990G8RBE1w== + dependencies: + "@tediousjs/connection-string" "^0.5.0" + commander "^11.0.0" + debug "^4.3.3" + rfdc "^1.3.0" + tarn "^3.0.2" + tedious "^18.2.1" + multicast-dns@^7.2.5: version "7.2.5" resolved "https://registry.yarnpkg.com/multicast-dns/-/multicast-dns-7.2.5.tgz#77eb46057f4d7adbd16d9290fa7299f6fa64cced" @@ -26702,6 +26709,22 @@ tarn@^3.0.1, tarn@^3.0.2: resolved "https://registry.yarnpkg.com/tarn/-/tarn-3.0.2.tgz#73b6140fbb881b71559c4f8bfde3d9a4b3d27693" integrity sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ== +tedious@*, tedious@^18.2.1: + version "18.6.1" + resolved "https://registry.yarnpkg.com/tedious/-/tedious-18.6.1.tgz#1c4a3f06c891be67a032117e2e25193286d44496" + integrity sha512-9AvErXXQTd6l7TDd5EmM+nxbOGyhnmdbp/8c3pw+tjaiSXW9usME90ET/CRG1LN1Y9tPMtz/p83z4Q97B4DDpw== + dependencies: + "@azure/core-auth" "^1.7.2" + "@azure/identity" "^4.2.1" + "@azure/keyvault-keys" "^4.4.0" + "@js-joda/core" "^5.6.1" + "@types/node" ">=18" + bl "^6.0.11" + iconv-lite "^0.6.3" + js-md4 "^0.3.2" + native-duplexpair "^1.0.0" + sprintf-js "^1.1.3" + tedious@^16.4.0: version "16.7.1" resolved "https://registry.yarnpkg.com/tedious/-/tedious-16.7.1.tgz#1190f30fd99a413f1dc9250dee4835cf0788b650"