[WIP] feat(clickhouse): Use ClickHouse query with parameters via HTTP interface

Author: mcheshkov

packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts

@@ -222,12 +222,41 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
       true;
   }
 
-  public async query(query: string, values: unknown[]) {
+  public async query(query: string, values?: unknown[]) {
     return this.queryResponse(query, values).then((res: any) => this.normaliseResponse(res));
   }
 
-  protected queryResponse(query: string, values: unknown[]) {
-    const formattedQuery = sqlstring.format(query, values);
+  protected queryResponse(query: string, values?: unknown[]) {
+    // todo drop this
+    console.log('queryResponse call', query, values);
+
+    // const formattedQuery = sqlstring.format(query, values);
+    const formattedQuery = query;
+
+    // `queryOptions` object will be merged into query params via querystring.stringify
+    // https://github.com/cube-js/apla-node-clickhouse/blob/5a6577fc97ba6911171753fc65b2cd2f6170f2f7/src/clickhouse.js#L347-L348
+    // https://github.com/cube-js/apla-node-clickhouse/blob/5a6577fc97ba6911171753fc65b2cd2f6170f2f7/src/clickhouse.js#L265-L266
+    // https://github.com/cube-js/apla-node-clickhouse/blob/5a6577fc97ba6911171753fc65b2cd2f6170f2f7/src/clickhouse.js#L336-L338
+    // https://github.com/cube-js/apla-node-clickhouse/blob/5a6577fc97ba6911171753fc65b2cd2f6170f2f7/src/clickhouse.js#L173-L175
+
+    // We can use `toSearchParams` or `formatQueryParams` from `@clickhouse/client-common` to prepare params, and extract only interesting ones
+    // Beware - these functions marked as "For implementations usage only - should not be re-exported", so, probably, ot could be moved or disappear completely
+    // https://github.com/ClickHouse/clickhouse-js/blob/a15cce93545c792852e34c05ce31954c75d11486/packages/client-common/src/utils/url.ts#L57-L61
+
+    // HTTP interface itself is documented, so it should be fine
+    // https://clickhouse.com/docs/en/interfaces/cli#cli-queries-with-parameters
+    // https://clickhouse.com/docs/en/interfaces/http#cli-queries-with-parameters
+
+    const paramsValues = Object.fromEntries((values ?? []).map((value, idx) => {
+      const paramName = this.paramName(idx);
+      const paramKey = `param_${paramName}`;
+      // TODO prepare value
+      const preparedValue = value;
+      return [paramKey, preparedValue];
+    }));
+
+    // todo drop this
+    console.log('queryResponse prepared', formattedQuery, paramsValues);
 
     return this.withConnection((connection, queryId) => connection.querying(formattedQuery, {
       dataObjects: true,
@@ -241,6 +270,9 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
         //
         //
         ...(this.readOnlyMode ? {} : { join_use_nulls: 1 }),
+
+        // Add parameter values to query string
+        ...paramsValues,
       }
     }));
   }
@@ -309,6 +341,15 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
     return [{ schema_name: this.config.queryOptions.database }];
   }
 
+  protected paramName(paramIndex: number): string {
+    return `p${paramIndex}`;
+  }
+
+  public param(paramIndex: number): string {
+    // TODO not always string
+    return `{${this.paramName(paramIndex)}:String}`;
+  }
+
   public async stream(
     query: string,
     values: unknown[],

packages/cubejs-clickhouse-driver/test/ClickHouseDriver.test.ts

@@ -58,13 +58,23 @@ describe('ClickHouseDriver', () => {
         []
       );
 
-      await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
+      function mkPlaceholdersTuple(len: number): string {
+        const parts = new Array(len).fill('').map((_, idx) => driver.param(idx));
+        return `(${parts.join(',')})`;
+      }
+
+      async function insert(table: string, values: Array<unknown>): Promise<void> {
+        const placeholders = mkPlaceholdersTuple(values.length);
+        await driver.query(`INSERT INTO ${table} VALUES ${placeholders}`, values);
+      }
+
+      await insert('test.types_test', [
         '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00.000', '2020-01-01 00:00:00.000000', '2020-01-01 00:00:00.000000000', 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.01, 1.01, 1.01, 'hello', 'world'
       ]);
-      await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
+      await insert('test.types_test', [
         '2020-01-02', '2020-01-02 00:00:00', '2020-01-02 00:00:00.123', '2020-01-02 00:00:00.123456', '2020-01-02 00:00:00.123456789', 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2.02, 2.02, 2.02, 'hello', 'world'
       ]);
-      await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
+      await insert('test.types_test', [
         '2020-01-03', '2020-01-03 00:00:00', '2020-01-03 00:00:00.234', '2020-01-03 00:00:00.234567', '2020-01-03 00:00:00.234567890', 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3.03, 3.03, 3.03, 'hello', 'world'
       ]);
     });

packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts

@@ -3,6 +3,7 @@ import { BaseQuery } from './BaseQuery';
 import { BaseFilter } from './BaseFilter';
 import { UserError } from '../compiler/UserError';
 import { BaseTimeDimension } from './BaseTimeDimension';
+import { ParamAllocator } from './ParamAllocator';
 
 const GRANULARITY_TO_INTERVAL = {
   day: 'Day',
@@ -14,6 +15,13 @@ const GRANULARITY_TO_INTERVAL = {
   year: 'Year',
 };
 
+class ClickHouseParamAllocator extends ParamAllocator {
+  public paramPlaceHolder(paramIndex: number): string {
+    // TODO not always string
+    return `{p${paramIndex}:String}`;
+  }
+}
+
 class ClickHouseFilter extends BaseFilter {
   public likeIgnoreCase(column, not, param, type) {
     const p = (!type || type === 'contains' || type === 'ends') ? '%' : '';
@@ -31,6 +39,10 @@ class ClickHouseFilter extends BaseFilter {
 }
 
 export class ClickHouseQuery extends BaseQuery {
+  public newParamAllocator(expressionParams) {
+    return new ClickHouseParamAllocator(expressionParams);
+  }
+
   public newFilter(filter) {
     return new ClickHouseFilter(this, filter);
   }
@@ -268,6 +280,8 @@ export class ClickHouseQuery extends BaseQuery {
 
   public sqlTemplates() {
     const templates = super.sqlTemplates();
+    // TODO not every param is a string
+    templates.params.param = '{p{{ param_index }}:String}';
     templates.functions.DATETRUNC = 'DATE_TRUNC({{ args_concat }})';
     // TODO: Introduce additional filter in jinja? or parseDateTimeBestEffort?
     // https://github.com/ClickHouse/ClickHouse/issues/19351