test(schema-compiler): Replace apla-clickhouse with @clickhouse/client

Author: mcheshkov

packages/cubejs-schema-compiler/package.json

@@ -57,7 +57,7 @@
     "uuid": "^8.3.2"
   },
   "devDependencies": {
-    "@cubejs-backend/apla-clickhouse": "^1.7.0",
+    "@clickhouse/client": "^1.7.0",
     "@cubejs-backend/linter": "^1.0.0",
     "@cubejs-backend/query-orchestrator": "1.1.7",
     "@types/babel__code-frame": "^7.0.6",

packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts

@@ -1,16 +1,12 @@
-/* eslint-disable */
-// @ts-ignore
-import ClickHouse from '@cubejs-backend/apla-clickhouse';
+import { createClient } from '@clickhouse/client';
+import type { ClickHouseClient, ResponseJSON } from '@clickhouse/client';
 import { GenericContainer } from 'testcontainers';
 import type { StartedTestContainer } from 'testcontainers';
 import { format as formatSql } from 'sqlstring';
 import { v4 as uuidv4 } from 'uuid';
 import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery';
 import { BaseDbRunner } from "../utils/BaseDbRunner";
 
-// Just a placeholder for now
-type ClickHouseClient = any;
-
 process.env.TZ = 'GMT';
 
 export class ClickHouseDbRunner extends BaseDbRunner {
@@ -35,27 +31,32 @@ export class ClickHouseDbRunner extends BaseDbRunner {
     // let engine = 'MergeTree PARTITION BY id ORDER BY (id) SETTINGS index_granularity = 8192'
     const engine = 'Memory';
 
-    await clickHouse.querying(`
-    CREATE TEMPORARY TABLE visitors (id UInt64, amount UInt64, created_at DateTime, updated_at DateTime, status UInt64, source Nullable(String), latitude Float64, longitude Float64)
-    ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    await clickHouse.command({ query: `
+      CREATE TEMPORARY TABLE visitors (id UInt64, amount UInt64, created_at DateTime, updated_at DateTime, status UInt64, source Nullable(String), latitude Float64, longitude Float64)
+      ENGINE = ${engine}
+    ` });
 
-    await clickHouse.querying(`
-    CREATE TEMPORARY TABLE visitor_checkins (id UInt64, visitor_id UInt64, created_at DateTime, source Nullable(String))
-    ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    await clickHouse.command({ query: `
+      CREATE TEMPORARY TABLE visitor_checkins (id UInt64, visitor_id UInt64, created_at DateTime, source Nullable(String))
+      ENGINE = ${engine}
+    ` });
 
-    await clickHouse.querying(`
-    CREATE TEMPORARY TABLE cards (id UInt64, visitor_id UInt64, visitor_checkin_id UInt64)
-    ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    await clickHouse.command({ query: `
+      CREATE TEMPORARY TABLE cards (id UInt64, visitor_id UInt64, visitor_checkin_id UInt64)
+      ENGINE = ${engine}
+    ` });
 
-    await clickHouse.querying(`
-    CREATE TEMPORARY TABLE events (id UInt64, type String, name String, started_at DateTime64, ended_at Nullable(DateTime64))
-    ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    await clickHouse.command({ query: `
+      CREATE TEMPORARY TABLE events (id UInt64, type String, name String, started_at DateTime64, ended_at Nullable(DateTime64))
+      ENGINE = ${engine}
+    ` });
 
-    await clickHouse.querying(`
-    CREATE TEMPORARY TABLE numbers (num Int)
-    ENGINE = ${engine}`, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    await clickHouse.command({ query: `
+      CREATE TEMPORARY TABLE numbers (num Int)
+      ENGINE = ${engine}
+    ` });
 
-    await clickHouse.querying(`
+    await clickHouse.command({ query: `
       INSERT INTO
       visitors
       (id, amount, created_at, updated_at, status, source, latitude, longitude) VALUES
@@ -65,9 +66,9 @@ export class ClickHouseDbRunner extends BaseDbRunner {
       (4, 400, '2017-01-06 16:00:00', '2017-01-24 16:00:00', 2, null, 120.120, 10.60),
       (5, 500, '2017-01-06 16:00:00', '2017-01-24 16:00:00', 2, null, 120.120, 58.10),
       (6, 500, '2016-09-06 16:00:00', '2016-09-06 16:00:00', 2, null, 120.120, 58.10)
-    `, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    ` });
 
-    await clickHouse.querying(`
+    await clickHouse.command({ query: `
       INSERT INTO
       visitor_checkins
       (id, visitor_id, created_at, source) VALUES
@@ -77,28 +78,28 @@ export class ClickHouseDbRunner extends BaseDbRunner {
       (4, 2, '2017-01-04 16:00:00', null),
       (5, 2, '2017-01-04 16:00:00', null),
       (6, 3, '2017-01-05 16:00:00', null)
-    `, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    ` });
 
-    await clickHouse.querying(`
+    await clickHouse.command({ query: `
       INSERT INTO
       cards
       (id, visitor_id, visitor_checkin_id) VALUES
       (1, 1, 1),
       (2, 1, 2),
       (3, 3, 6)
-    `, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    ` });
 
-    await clickHouse.querying(`
+    await clickHouse.command({ query: `
       INSERT INTO
       events
       (id, type, name, started_at, ended_at) VALUES
       (1, 'moon_missions', 'Apollo 10', '1969-05-18 16:49:00', '1969-05-26 16:52:23'),
       (2, 'moon_missions', 'Apollo 11', '1969-07-16 13:32:00', '1969-07-24 16:50:35'),
       (3, 'moon_missions', 'Artemis I', '2021-11-16 06:32:00', '2021-12-11 18:50:00'),
       (4, 'private_missions', 'Axiom Mission 1', '2022-04-08 15:17:12', '2022-04-25 17:06:00')
-    `, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    ` });
 
-    await clickHouse.querying(`
+    await clickHouse.command({ query: `
       INSERT INTO
       numbers
       (num) VALUES
@@ -108,7 +109,7 @@ export class ClickHouseDbRunner extends BaseDbRunner {
       (30), (31), (32), (33), (34), (35), (36), (37), (38), (39),
       (40), (41), (42), (43), (44), (45), (46), (47), (48), (49),
       (50), (51), (52), (53), (54), (55), (56), (57), (58), (59)
-    `, { queryOptions: { session_id: clickHouse.sessionId, join_use_nulls: '1' } });
+    ` });
   }
 
   public override async testQueries(queries: Array<[string, Array<unknown>]>, prepareDataSet?: ((client: ClickHouseClient) => Promise<void>) | null): Promise<Array<Array<Record<string, unknown>>>> {
@@ -127,18 +128,17 @@ export class ClickHouseDbRunner extends BaseDbRunner {
       port = this.container.getMappedPort(8123);
     }
 
-    const clickHouse = new ClickHouse({
-      host,
-      port,
-    });
+    const clickHouse = createClient({
+      url: `http://${host}:${port}`,
 
-    clickHouse.sessionId = uuidv4(); // needed for tests to use temporary tables
+      // needed for tests to use temporary tables
+      session_id: uuidv4(),
+      max_open_connections: 1,
+    });
 
     prepareDataSet = prepareDataSet || this.gutterDataSet;
     await prepareDataSet(clickHouse);
 
-    const requests: Array<unknown> = [];
-
     // Controls whether functions return results with extended date and time ranges.
     //
     // 0 — Functions return Date or DateTime for all arguments (default).
@@ -150,19 +150,23 @@ export class ClickHouseDbRunner extends BaseDbRunner {
     //
     // https://clickhouse.com/docs/en/operations/settings/settings#enable-extended-results-for-datetime-functions
     const extendedDateTimeResultsOptions = this.supportsExtendedDateTimeResults ? {
-      enable_extended_results_for_datetime_functions: '1'
-    } : {};
-
-    for (const [query, params] of queries) {
-      requests.push(clickHouse.querying(formatSql(query, params), {
-        dataObjects: true,
-        queryOptions: {
-          session_id: clickHouse.sessionId,
-          join_use_nulls: '1',
-          ...extendedDateTimeResultsOptions
-        }
-      }));
-    }
+      enable_extended_results_for_datetime_functions: 1
+    } as const : {};
+
+    const requests = queries
+      .map(async ([query, params]) => {
+        const resultSet = await clickHouse.query({
+          query: formatSql(query, params),
+          format: 'JSON',
+          clickhouse_settings: {
+            join_use_nulls: 1,
+            ...extendedDateTimeResultsOptions
+          }
+        });
+        // Because we used JSON format we expect each row in result set to be a record of column name => value
+        const result = await resultSet.json<Record<string, unknown>>();
+        return result;
+      });
 
     const results = await Promise.all(requests);
 
@@ -189,26 +193,37 @@ export class ClickHouseDbRunner extends BaseDbRunner {
   //
   //  https://github.com/statsbotco/cube.js/pull/98#discussion_r279698399
   //
-  protected static _normaliseResponse(res: any): Array<Record<string, unknown>> {
-    if (process.env.DEBUG_LOG === 'true') console.log(res);
-    if (res.data) {
-      res.data.forEach(row => {
-        for (const field in row) {
-          const value = row[field];
-          if (value !== null) {
-            const meta = res.meta.find(m => m.name == field);
-            if (meta.type.includes('DateTime')) {
-              row[field] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
-            } else if (meta.type.includes('Date')) {
-              row[field] = `${value}T00:00:00.000`;
-            } else if (meta.type.includes('Int') || meta.type.includes('Float')) {
-              // convert all numbers into strings
-              row[field] = `${value}`;
+  protected static _normaliseResponse(res: ResponseJSON<Record<string, unknown>>): Array<Record<string, unknown>> {
+    if (process.env.DEBUG_LOG === 'true') {
+      console.log(res);
+    }
+
+    const { meta, data } = res;
+    if (meta === undefined) {
+      throw new Error('Unexpected missing meta');
+    }
+
+    data.forEach(row => {
+      for (const [field, value] of Object.entries(row)) {
+        if (value !== null) {
+          const fieldMeta = meta.find(m => m.name == field);
+          if (fieldMeta === undefined) {
+            throw new Error(`Missing meta for field ${field}`);
+          }
+          if (fieldMeta.type.includes('DateTime')) {
+            if (typeof value !== 'string') {
+              throw new Error(`Unexpected value for ${field}`);
             }
+            row[field] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
+          } else if (fieldMeta.type.includes('Date')) {
+            row[field] = `${value}T00:00:00.000`;
+          } else if (fieldMeta.type.includes('Int') || fieldMeta.type.includes('Float')) {
+            // convert all numbers into strings
+            row[field] = `${value}`;
           }
         }
-      });
-    }
-    return res.data;
+      }
+    });
+    return data;
   }
 }

yarn.lock

@@ -4365,11 +4365,6 @@
     tiny-invariant "^1.3.3"
     valid-url "^1.0.9"
 
-"@cubejs-backend/apla-clickhouse@^1.7.0":
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/@cubejs-backend/apla-clickhouse/-/apla-clickhouse-1.7.0.tgz#6359f46c56492d1704d18be0210c7546fdac5f5e"
-  integrity sha512-qwXapTC/qosA6RprElRjnl8gmlDQaxtJPtbgcdjyNvkmiyao1HI+w5QkjHWCiVm6aTzE0gjFr6/2y87TZ9fojg==
-
 "@cubejs-backend/dotenv@^9.0.2":
   version "9.0.2"
   resolved "https://registry.yarnpkg.com/@cubejs-backend/dotenv/-/dotenv-9.0.2.tgz#c3679091b702f0fd38de120c5a63943fcdc0dcbf"