[WIP] feat(clickhouse): Use ClickHouse query with parameters via HTTP interface

Author: mcheshkov

packages/cubejs-clickhouse-driver/package.json

@@ -27,6 +27,7 @@
     "integration:clickhouse": "jest dist/test"
   },
   "dependencies": {
+    "@clickhouse/client-common": "^1.8.0",
     "@cubejs-backend/apla-clickhouse": "^1.7",
     "@cubejs-backend/base-driver": "1.1.3",
     "@cubejs-backend/shared": "1.1.3",

packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts

@@ -23,7 +23,9 @@ import {
 } from '@cubejs-backend/base-driver';
 import genericPool, { Pool } from 'generic-pool';
 import { v4 as uuidv4 } from 'uuid';
+// TODO get rid of sqlstring completely
 import sqlstring from 'sqlstring';
+import { formatQueryParams } from '@clickhouse/client-common';
 
 import { HydrationStream, transformRow } from './HydrationStream';
 
@@ -222,12 +224,42 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
       true;
   }
 
-  public async query(query: string, values: unknown[]) {
-    return this.queryResponse(query, values).then((res: any) => this.normaliseResponse(res));
+  public async query(query: string, values?: unknown[]) {
+    return this.queryResponse(query, values).then((res: any) => this.normaliseResponse(res)).catch((err: unknown) => {
+      throw new Error(`Failed during query; query: ${query}; values: ${values}`, { cause: err });
+    });
   }
 
-  protected queryResponse(query: string, values: unknown[]) {
-    const formattedQuery = sqlstring.format(query, values);
+  protected queryResponse(query: string, values?: unknown[]) {
+    // todo drop this
+    console.log('queryResponse call', query, values);
+
+    // const formattedQuery = sqlstring.format(query, values);
+    const formattedQuery = query;
+
+    // `queryOptions` object will be merged into query params via querystring.stringify
+    // https://github.com/cube-js/apla-node-clickhouse/blob/5a6577fc97ba6911171753fc65b2cd2f6170f2f7/src/clickhouse.js#L347-L348
+    // https://github.com/cube-js/apla-node-clickhouse/blob/5a6577fc97ba6911171753fc65b2cd2f6170f2f7/src/clickhouse.js#L265-L266
+    // https://github.com/cube-js/apla-node-clickhouse/blob/5a6577fc97ba6911171753fc65b2cd2f6170f2f7/src/clickhouse.js#L336-L338
+    // https://github.com/cube-js/apla-node-clickhouse/blob/5a6577fc97ba6911171753fc65b2cd2f6170f2f7/src/clickhouse.js#L173-L175
+
+    // We can use `toSearchParams` or `formatQueryParams` from `@clickhouse/client-common` to prepare params, and extract only interesting ones
+    // Beware - these functions marked as "For implementations usage only - should not be re-exported", so, probably, ot could be moved or disappear completely
+    // https://github.com/ClickHouse/clickhouse-js/blob/a15cce93545c792852e34c05ce31954c75d11486/packages/client-common/src/utils/url.ts#L57-L61
+
+    // HTTP interface itself is documented, so it should be fine
+    // https://clickhouse.com/docs/en/interfaces/cli#cli-queries-with-parameters
+    // https://clickhouse.com/docs/en/interfaces/http#cli-queries-with-parameters
+
+    const paramsValues = Object.fromEntries((values ?? []).map((value, idx) => {
+      const paramName = this.paramName(idx);
+      const paramKey = `param_${paramName}`;
+      const preparedValue = formatQueryParams(value);
+      return [paramKey, preparedValue];
+    }));
+
+    // todo drop this
+    console.log('queryResponse prepared', formattedQuery, paramsValues);
 
     return this.withConnection((connection, queryId) => connection.querying(formattedQuery, {
       dataObjects: true,
@@ -241,6 +273,9 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
         //
         //
         ...(this.readOnlyMode ? {} : { join_use_nulls: 1 }),
+
+        // Add parameter values to query string
+        ...paramsValues,
       }
     }));
   }
@@ -309,6 +344,15 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
     return [{ schema_name: this.config.queryOptions.database }];
   }
 
+  protected paramName(paramIndex: number): string {
+    return `p${paramIndex}`;
+  }
+
+  public param(paramIndex: number): string {
+    // TODO not always string
+    return `{${this.paramName(paramIndex)}:String}`;
+  }
+
   public async stream(
     query: string,
     values: unknown[],
@@ -416,7 +460,7 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
   }
 
   public getTablesQuery(schemaName: string) {
-    return this.query('SELECT name as table_name FROM system.tables WHERE database = ?', [schemaName]);
+    return this.query('SELECT name as table_name FROM system.tables WHERE database = {p0:String}', [schemaName]);
   }
 
   protected getExportBucket(

packages/cubejs-clickhouse-driver/test/ClickHouseDriver.test.ts

@@ -58,13 +58,23 @@ describe('ClickHouseDriver', () => {
         []
       );
 
-      await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
+      function mkPlaceholdersTuple(len: number): string {
+        const parts = new Array(len).fill('').map((_, idx) => driver.param(idx));
+        return `(${parts.join(',')})`;
+      }
+
+      async function insert(table: string, values: Array<unknown>): Promise<void> {
+        const placeholders = mkPlaceholdersTuple(values.length);
+        await driver.query(`INSERT INTO ${table} VALUES ${placeholders}`, values);
+      }
+
+      await insert('test.types_test', [
         '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00.000', '2020-01-01 00:00:00.000000', '2020-01-01 00:00:00.000000000', 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.01, 1.01, 1.01, 'hello', 'world'
       ]);
-      await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
+      await insert('test.types_test', [
         '2020-01-02', '2020-01-02 00:00:00', '2020-01-02 00:00:00.123', '2020-01-02 00:00:00.123456', '2020-01-02 00:00:00.123456789', 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2.02, 2.02, 2.02, 'hello', 'world'
       ]);
-      await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
+      await insert('test.types_test', [
         '2020-01-03', '2020-01-03 00:00:00', '2020-01-03 00:00:00.234', '2020-01-03 00:00:00.234567', '2020-01-03 00:00:00.234567890', 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3.03, 3.03, 3.03, 'hello', 'world'
       ]);
     });
@@ -205,8 +215,8 @@ describe('ClickHouseDriver', () => {
       try {
         await driver.createSchemaIfNotExists(name);
         await driver.query(`CREATE TABLE ${name}.test (x Int32, s String) ENGINE Log`, []);
-        await driver.query(`INSERT INTO ${name}.test VALUES (?, ?), (?, ?), (?, ?)`, [1, 'str1', 2, 'str2', 3, 'str3']);
-        const values = await driver.query(`SELECT * FROM ${name}.test WHERE x = ?`, [2]);
+        await driver.query(`INSERT INTO ${name}.test VALUES ({p0:Int32}, {p1:String}), ({p2:Int32}, {p3:String}), ({p4:Int32}, {p5:String})`, [1, 'str1', 2, 'str2', 3, 'str3']);
+        const values = await driver.query(`SELECT * FROM ${name}.test WHERE x = {p0:Int32}`, [2]);
         expect(values).toEqual([{ x: '2', s: 'str2' }]);
       } finally {
         await driver.query(`DROP DATABASE ${name}`, []);
@@ -220,10 +230,10 @@ describe('ClickHouseDriver', () => {
       try {
         await driver.createSchemaIfNotExists(name);
         await driver.query(`CREATE TABLE ${name}.a (x Int32, s String) ENGINE Log`, []);
-        await driver.query(`INSERT INTO ${name}.a VALUES (?, ?), (?, ?), (?, ?)`, [1, 'str1', 2, 'str2', 3, 'str3']);
+        await driver.query(`INSERT INTO ${name}.a VALUES ({p0:Int32}, {p1:String}), ({p2:Int32}, {p3:String}), ({p4:Int32}, {p5:String})`, [1, 'str1', 2, 'str2', 3, 'str3']);
 
         await driver.query(`CREATE TABLE ${name}.b (x Int32, s String) ENGINE Log`, []);
-        await driver.query(`INSERT INTO ${name}.b VALUES (?, ?), (?, ?), (?, ?)`, [2, 'str2', 3, 'str3', 4, 'str4']);
+        await driver.query(`INSERT INTO ${name}.b VALUES ({p0:Int32}, {p1:String}), ({p2:Int32}, {p3:String}), ({p4:Int32}, {p5:String})`, [2, 'str2', 3, 'str3', 4, 'str4']);
 
         const values = await driver.query(`SELECT * FROM ${name}.a LEFT OUTER JOIN ${name}.b ON a.x = b.x`, []);
         expect(values).toEqual([
@@ -245,7 +255,7 @@ describe('ClickHouseDriver', () => {
 
   it('datetime with specific timezone', async () => {
     await doWithDriver(async (driver) => {
-      const rows = await driver.query('SELECT toDateTime(?, \'Asia/Istanbul\') as dt', [
+      const rows = await driver.query('SELECT toDateTime({p0:String}, \'Asia/Istanbul\') as dt', [
         '2020-01-01 00:00:00'
       ]);
       expect(rows).toEqual([{

packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts

@@ -3,6 +3,7 @@ import { BaseQuery } from './BaseQuery';
 import { BaseFilter } from './BaseFilter';
 import { UserError } from '../compiler/UserError';
 import { BaseTimeDimension } from './BaseTimeDimension';
+import { ParamAllocator } from './ParamAllocator';
 
 const GRANULARITY_TO_INTERVAL = {
   day: 'Day',
@@ -14,6 +15,13 @@ const GRANULARITY_TO_INTERVAL = {
   year: 'Year',
 };
 
+class ClickHouseParamAllocator extends ParamAllocator {
+  public paramPlaceHolder(paramIndex: number): string {
+    // TODO not always string
+    return `{p${paramIndex}:String}`;
+  }
+}
+
 class ClickHouseFilter extends BaseFilter {
   public likeIgnoreCase(column, not, param, type) {
     const p = (!type || type === 'contains' || type === 'ends') ? '%' : '';
@@ -31,6 +39,10 @@ class ClickHouseFilter extends BaseFilter {
 }
 
 export class ClickHouseQuery extends BaseQuery {
+  public newParamAllocator(expressionParams) {
+    return new ClickHouseParamAllocator(expressionParams);
+  }
+
   public newFilter(filter) {
     return new ClickHouseFilter(this, filter);
   }
@@ -268,6 +280,8 @@ export class ClickHouseQuery extends BaseQuery {
 
   public sqlTemplates() {
     const templates = super.sqlTemplates();
+    // TODO not every param is a string
+    templates.params.param = '{p{{ param_index }}:String}';
     templates.functions.DATETRUNC = 'DATE_TRUNC({{ args_concat }})';
     // TODO: Introduce additional filter in jinja? or parseDateTimeBestEffort?
     // https://github.com/ClickHouse/ClickHouse/issues/19351

yarn.lock

@@ -4222,6 +4222,11 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@clickhouse/client-common@^1.8.0":
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/@clickhouse/client-common/-/client-common-1.8.0.tgz#6775c4f1f4b56f393cbacec0a1b544dafcf8d5ed"
+  integrity sha512-aQgH0UODGuFHfL8rgeLSrGCoh3NCoNUs0tFGl0o79iyfASfvWtT/K/X3RM0QJpXXOgXpB//T2nD5XvCFtdk32w==
+
 "@codemirror/highlight@^0.19.0":
   version "0.19.6"
   resolved "https://registry.yarnpkg.com/@codemirror/highlight/-/highlight-0.19.6.tgz#7f2e066f83f5649e8e0748a3abe0aaeaf64b8ac2"