Skip to content

Commit 401e9e1

Browse files
ovrovr
authored
fix(clickhouse-driver): Initial support for DateTime64, fix #7537 (#7538)
1 parent 75e201d commit 401e9e1

File tree

8 files changed

+99
-80
lines changed

8 files changed

+99
-80
lines changed

‎.github/actions/integration/clickhouse.sh‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,23 @@ set -eo pipefail
44
# Debug log for test containers
55
export DEBUG=testcontainers
66

7-
export TEST_CLICKHOUSE_VERSION=21.1.2
7+
export TEST_CLICKHOUSE_VERSION=23.11
88

99
echo "::group::Clickhouse ${TEST_CLICKHOUSE_VERSION}";
10-
docker pull yandex/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
10+
docker pull clickhouse/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
1111
yarn lerna run --concurrency 1 --stream --no-prefix integration:clickhouse
1212
echo "::endgroup::"
1313

14-
export TEST_CLICKHOUSE_VERSION=20.6
14+
export TEST_CLICKHOUSE_VERSION=22.8
1515

1616
echo "::group::Clickhouse ${TEST_CLICKHOUSE_VERSION}";
17-
docker pull yandex/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
17+
docker pull clickhouse/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
1818
yarn lerna run --concurrency 1 --stream --no-prefix integration:clickhouse
1919
echo "::endgroup::"
2020

21-
export TEST_CLICKHOUSE_VERSION=19
21+
export TEST_CLICKHOUSE_VERSION=21.8
2222

2323
echo "::group::Clickhouse ${TEST_CLICKHOUSE_VERSION}";
24-
docker pull yandex/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
24+
docker pull clickhouse/clickhouse-server:${TEST_CLICKHOUSE_VERSION}
2525
yarn lerna run --concurrency 1 --stream --no-prefix integration:clickhouse
2626
echo "::endgroup::"

‎packages/cubejs-clickhouse-driver/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,16 @@
3131
"@cubejs-backend/base-driver": "^0.34.33",
3232
"@cubejs-backend/shared": "^0.34.33",
3333
"generic-pool": "^3.6.0",
34+
"moment": "^2.24.0",
3435
"sqlstring": "^2.3.1",
3536
"uuid": "^8.3.2"
3637
},
3738
"license": "Apache-2.0",
3839
"devDependencies": {
40+
"@cubejs-backend/testing-shared": "^0.34.35",
3941
"@cubejs-backend/linter": "^0.34.25",
4042
"@types/jest": "^27",
4143
"jest": "27",
42-
"stream-to-array": "^2.3.0",
43-
"testcontainers": "^8.12",
4444
"typescript": "~5.2.2"
4545
},
4646
"publishConfig": {

‎packages/cubejs-clickhouse-driver/src/ClickHouseDriver.ts‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ import {
2121
import genericPool, { Pool } from 'generic-pool';
2222
import { v4 as uuidv4 } from 'uuid';
2323
import sqlstring from 'sqlstring';
24-
import { HydrationStream } from './HydrationStream';
24+
import * as moment from 'moment';
25+
26+
import { HydrationStream, transformRow } from './HydrationStream';
2527

2628
const ClickHouse = require('@apla/clickhouse');
2729

@@ -225,30 +227,14 @@ export class ClickHouseDriver extends BaseDriver implements DriverInterface {
225227
}
226228

227229
protected normaliseResponse(res: any) {
228-
//
229-
//
230-
// ClickHouse returns DateTime as strings in format "YYYY-DD-MM HH:MM:SS"
231-
// cube.js expects them in format "YYYY-DD-MMTHH:MM:SS.000", so translate them based on the metadata returned
232-
//
233-
// ClickHouse returns some number types as js numbers, others as js string, normalise them all to strings
234-
//
235-
//
236230
if (res.data) {
231+
const meta = res.meta.reduce(
232+
(state: any, element: any) => ({ [element.name]: element, ...state }),
233+
{}
234+
);
235+
237236
res.data.forEach((row: any) => {
238-
Object.keys(row).forEach(field => {
239-
const value = row[field];
240-
if (value !== null) {
241-
const meta = res.meta.find((m: any) => m.name === field);
242-
if (meta.type.includes('DateTime')) {
243-
row[field] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
244-
} else if (meta.type.includes('Date')) {
245-
row[field] = `${value}T00:00:00.000`;
246-
} else if (meta.type.includes('Int') || meta.type.includes('Float') || meta.type.includes('Decimal')) {
247-
// convert all numbers into strings
248-
row[field] = `${value}`;
249-
}
250-
}
251-
});
237+
transformRow(row, meta);
252238
});
253239
}
254240
return res.data;

‎packages/cubejs-clickhouse-driver/src/HydrationStream.ts‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,37 @@
1-
/* eslint-disable no-restricted-syntax */
21
import stream, { TransformCallback } from 'stream';
2+
import * as moment from 'moment';
33

4-
export type HydrationMap = Record<string, any>;
4+
// ClickHouse returns DateTime as strings in format "YYYY-DD-MM HH:MM:SS"
5+
// cube.js expects them in format "YYYY-DD-MMTHH:MM:SS.000", so translate them based on the metadata returned
6+
//
7+
// ClickHouse returns some number types as js numbers, others as js string, normalise them all to strings
8+
export function transformRow(row: Record<string, any>, meta: any) {
9+
for (const [fieldName, value] of Object.entries(row)) {
10+
if (value !== null) {
11+
const metaForField = meta[fieldName];
12+
if (metaForField.type === 'DateTime') {
13+
row[fieldName] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
14+
} else if (metaForField.type.includes('DateTime64')) {
15+
row[fieldName] = moment.utc(value).format(moment.HTML5_FMT.DATETIME_LOCAL_MS);
16+
} else if (metaForField.type.includes('Date')) {
17+
row[fieldName] = `${value}T00:00:00.000`;
18+
} else if (metaForField.type.includes('Int')
19+
|| metaForField.type.includes('Float')
20+
|| metaForField.type.includes('Decimal')
21+
) {
22+
// convert all numbers into strings
23+
row[fieldName] = `${value}`;
24+
}
25+
}
26+
}
27+
}
528

629
export class HydrationStream extends stream.Transform {
730
public constructor(meta: any) {
831
super({
932
objectMode: true,
1033
transform(row: any[], encoding: BufferEncoding, callback: TransformCallback) {
11-
for (const [index, value] of Object.entries(row)) {
12-
if (value !== null) {
13-
const metaForField = meta[index];
14-
if (metaForField.type.includes('DateTime')) {
15-
row[<any>index] = `${value.substring(0, 10)}T${value.substring(11, 22)}.000`;
16-
} else if (metaForField.type.includes('Date')) {
17-
row[<any>index] = `${value}T00:00:00.000`;
18-
} else if (metaForField.type.includes('Int')
19-
|| metaForField.type.includes('Float')
20-
|| metaForField.type.includes('Decimal')
21-
) {
22-
// convert all numbers into strings
23-
row[<any>index] = `${value}`;
24-
}
25-
}
26-
}
34+
transformRow(row, meta);
2735

2836
this.push(row);
2937
callback();

‎packages/cubejs-clickhouse-driver/test/ClickHouseDriver.test.ts‎

Lines changed: 27 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
import { GenericContainer } from 'testcontainers';
1+
import { ClickhouseDBRunner } from '@cubejs-backend/testing-shared';
2+
import { streamToArray } from '@cubejs-backend/shared';
23

3-
import { ClickHouseDriver } from '../src/ClickHouseDriver';
4-
5-
const streamToArray = require('stream-to-array');
4+
import { ClickHouseDriver } from '../src';
65

76
describe('ClickHouseDriver', () => {
7+
jest.setTimeout(20 * 1000);
8+
89
let container: any;
910
let config: any;
1011

@@ -20,13 +21,7 @@ describe('ClickHouseDriver', () => {
2021

2122
// eslint-disable-next-line func-names
2223
beforeAll(async () => {
23-
jest.setTimeout(20 * 1000);
24-
25-
const version = process.env.TEST_CLICKHOUSE_VERSION || 'latest';
26-
27-
container = await new GenericContainer(`yandex/clickhouse-server:${version}`)
28-
.withExposedPorts(8123)
29-
.start();
24+
container = await ClickhouseDBRunner.startContainer({});
3025

3126
config = {
3227
host: 'localhost',
@@ -35,13 +30,14 @@ describe('ClickHouseDriver', () => {
3530

3631
await doWithDriver(async (driver) => {
3732
await driver.createSchemaIfNotExists('test');
38-
// Unsupported in old servers
39-
// datetime64 DateTime64,
4033
await driver.query(
4134
`
4235
CREATE TABLE test.types_test (
4336
date Date,
4437
datetime DateTime,
38+
datetime64_millis DateTime64(3, 'UTC'),
39+
datetime64_micros DateTime64(6, 'UTC'),
40+
datetime64_nanos DateTime64(9, 'UTC'),
4541
int8 Int8,
4642
int16 Int16,
4743
int32 Int32,
@@ -60,17 +56,17 @@ describe('ClickHouseDriver', () => {
6056
[]
6157
);
6258

63-
await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
64-
'2020-01-01', '2020-01-01 00:00:00', 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.01, 1.01, 1.01
59+
await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
60+
'2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00.000', '2020-01-01 00:00:00.000000', '2020-01-01 00:00:00.000000000', 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.01, 1.01, 1.01
6561
]);
66-
await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
67-
'2020-01-02', '2020-01-02 00:00:00', 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2.02, 2.02, 2.02
62+
await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
63+
'2020-01-02', '2020-01-02 00:00:00', '2020-01-02 00:00:00.123', '2020-01-02 00:00:00.123456', '2020-01-02 00:00:00.123456789', 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2.02, 2.02, 2.02
6864
]);
69-
await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
70-
'2020-01-03', '2020-01-03 00:00:00', 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3.03, 3.03, 3.03
65+
await driver.query('INSERT INTO test.types_test VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [
66+
'2020-01-03', '2020-01-03 00:00:00', '2020-01-03 00:00:00.234', '2020-01-03 00:00:00.234567', '2020-01-03 00:00:00.234567890', 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3.03, 3.03, 3.03
7167
]);
7268
});
73-
});
69+
}, 30 * 1000);
7470

7571
// eslint-disable-next-line func-names
7672
afterAll(async () => {
@@ -83,7 +79,7 @@ describe('ClickHouseDriver', () => {
8379
if (container) {
8480
await container.stop();
8581
}
86-
});
82+
}, 30 * 1000);
8783

8884
it('should construct', async () => {
8985
await doWithDriver(async () => {
@@ -160,8 +156,9 @@ describe('ClickHouseDriver', () => {
160156
expect(values).toEqual([{
161157
date: '2020-01-01T00:00:00.000',
162158
datetime: '2020-01-01T00:00:00.000',
163-
// Unsupported in old servers
164-
// datetime64: '2020-01-01T00:00:00.00.000',
159+
datetime64_millis: '2020-01-01T00:00:00.000',
160+
datetime64_micros: '2020-01-01T00:00:00.000',
161+
datetime64_nanos: '2020-01-01T00:00:00.000',
165162
int8: '1',
166163
int16: '1',
167164
int32: '1',
@@ -252,8 +249,9 @@ describe('ClickHouseDriver', () => {
252249
expect(tableData.types).toEqual([
253250
{ name: 'date', type: 'date' },
254251
{ name: 'datetime', type: 'timestamp' },
255-
// Unsupported in old servers
256-
// { name: 'datetime64', type: 'timestamp' },
252+
{ name: 'datetime64_millis', type: 'timestamp' },
253+
{ name: 'datetime64_micros', type: 'timestamp' },
254+
{ name: 'datetime64_nanos', type: 'timestamp' },
257255
{ name: 'int8', type: 'int' },
258256
{ name: 'int16', type: 'int' },
259257
{ name: 'int32', type: 'int' },
@@ -268,10 +266,10 @@ describe('ClickHouseDriver', () => {
268266
{ name: 'decimal64', type: 'decimal' },
269267
{ name: 'decimal128', type: 'decimal' },
270268
]);
271-
expect(await streamToArray(tableData.rowStream)).toEqual([
272-
['2020-01-01T00:00:00.000', '2020-01-01T00:00:00.000', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1.01', '1.01', '1.01'],
273-
['2020-01-02T00:00:00.000', '2020-01-02T00:00:00.000', '2', '2', '2', '2', '2', '2', '2', '2', '2', '2', '2.02', '2.02', '2.02'],
274-
['2020-01-03T00:00:00.000', '2020-01-03T00:00:00.000', '3', '3', '3', '3', '3', '3', '3', '3', '3', '3', '3.03', '3.03', '3.03'],
269+
expect(await streamToArray(tableData.rowStream as any)).toEqual([
270+
['2020-01-01T00:00:00.000', '2020-01-01T00:00:00.000', '2020-01-01T00:00:00.000', '2020-01-01T00:00:00.000', '2020-01-01T00:00:00.000', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1.01', '1.01', '1.01'],
271+
['2020-01-02T00:00:00.000', '2020-01-02T00:00:00.000', '2020-01-02T00:00:00.123', '2020-01-02T00:00:00.123', '2020-01-02T00:00:00.123', '2', '2', '2', '2', '2', '2', '2', '2', '2', '2', '2.02', '2.02', '2.02'],
272+
['2020-01-03T00:00:00.000', '2020-01-03T00:00:00.000', '2020-01-03T00:00:00.234', '2020-01-03T00:00:00.234', '2020-01-03T00:00:00.234', '3', '3', '3', '3', '3', '3', '3', '3', '3', '3', '3.03', '3.03', '3.03'],
275273
]);
276274
} finally {
277275
// @ts-ignore

‎packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,9 @@ export class ClickHouseDbRunner {
6868

6969
testQueries = async (queries, prepareDataSet) => {
7070
if (!this.container && !process.env.TEST_CLICKHOUSE_HOST) {
71-
const version = process.env.TEST_CLICKHOUSE_VERSION || '21.1.2';
71+
const version = process.env.TEST_CLICKHOUSE_VERSION || '23.11';
7272

73-
this.container = await new GenericContainer(`yandex/clickhouse-server:${version}`)
73+
this.container = await new GenericContainer(`clickhouse/clickhouse-server:${version}`)
7474
.withExposedPorts(8123)
7575
.start();
7676
}

‎packages/cubejs-testing-shared/src/db/clickhouse.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import { GenericContainer, Wait } from 'testcontainers';
2+
3+
import { DbRunnerAbstract, DBRunnerContainerOptions } from './db-runner.abstract';
4+
5+
type ClickhouseStartOptions = DBRunnerContainerOptions & {
6+
version?: string,
7+
};
8+
9+
export class ClickhouseDBRunner extends DbRunnerAbstract {
10+
public static startContainer(options: ClickhouseStartOptions) {
11+
const version = process.env.TEST_CLICKHOUSE_VERSION || options.version || '23.11';
12+
13+
const container = new GenericContainer(`clickhouse/clickhouse-server:${version}`)
14+
.withExposedPorts(8123)
15+
.withStartupTimeout(10 * 1000);
16+
17+
if (options.volumes) {
18+
// eslint-disable-next-line no-restricted-syntax
19+
for (const { source, target, bindMode } of options.volumes) {
20+
container.withBindMount(source, target, bindMode);
21+
}
22+
}
23+
24+
return container.start();
25+
}
26+
}

‎packages/cubejs-testing-shared/src/db/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
export * from './mysql';
22
export * from './postgres';
33
export * from './cubestore';
4+
export * from './clickhouse';
45
export * from './questdb';
56
export * from './materialize';
67
export * from './crate';

0 commit comments

Comments
 (0)