Clickhouse string ordering and string filtering by UTF8 instead of bytes

packages/cubejs-backend-shared/src/env.ts

@@ -1169,6 +1169,53 @@ const variables: Record<string, (...args: any) => any> = {
     ]
   ),
 
+  /**
+   * ClickHouse sort collation.
+   */
+  clickhouseSortCollation: ({ dataSource }: {dataSource: string }) => {
+    const val = process.env[
+      keyByDataSource('CUBEJS_DB_CLICKHOUSE_SORT_COLLATION', dataSource)
+    ];
+    if (!val) {
+      // Default to 'en' collation
+      return 'en';
+    }
+    return val;
+  },
+
+  /**
+   * Clickhouse use collation flag.
+   */
+
+  clickhouseUseCollation: ({ dataSource }: { dataSource: string }) => {
+    const val = process.env[
+      keyByDataSource(
+        'CUBEJS_DB_CLICKHOUSE_USE_COLLATION',
+        dataSource,
+      )
+    ];
+
+    if (val) {
+      if (val.toLocaleLowerCase() === 'true') {
+        return true;
+      } else if (val.toLowerCase() === 'false') {
+        return false;
+      } else {
+        throw new TypeError(
+          `The ${
+            keyByDataSource(
+              'CUBEJS_DB_CLICKHOUSE_USE_COLLATION',
+              dataSource,
+            )
+          } must be either 'true' or 'false'.`
+        );
+      }
+    } else {
+      // Default to true
+      return true;
+    }
+  },
+
   /** ****************************************************************
    * ElasticSearch Driver                                            *
    ***************************************************************** */

packages/cubejs-backend-shared/test/db_env_multi.test.ts

@@ -1539,6 +1539,77 @@ describe('Multiple datasources', () => {
     );
   });
 
+  test('getEnv("clickhouseSortCollation")', () => {
+    process.env.CUBEJS_DB_CLICKHOUSE_SORT_COLLATION = 'default1';
+    process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_SORT_COLLATION = 'postgres1';
+    process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_SORT_COLLATION = 'wrong1';
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'default' })).toEqual('default1');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'postgres' })).toEqual('postgres1');
+    expect(() => getEnv('clickhouseSortCollation', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+
+    process.env.CUBEJS_DB_CLICKHOUSE_SORT_COLLATION = 'default2';
+    process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_SORT_COLLATION = 'postgres2';
+    process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_SORT_COLLATION = 'wrong2';
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'default' })).toEqual('default2');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'postgres' })).toEqual('postgres2');
+    expect(() => getEnv('clickhouseSortCollation', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+
+    delete process.env.CUBEJS_DB_CLICKHOUSE_SORT_COLLATION;
+    delete process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_SORT_COLLATION;
+    delete process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_SORT_COLLATION;
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'default' })).toEqual('en');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'postgres' })).toEqual('en');
+    expect(() => getEnv('clickhouseSortCollation', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+  });
+
+  test('getEnv("clickhouseUseCollation")', () => {
+    process.env.CUBEJS_DB_CLICKHOUSE_USE_COLLATION = 'true';
+    process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_USE_COLLATION = 'true';
+    process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_USE_COLLATION = 'true';
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'default' })).toEqual(true);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'postgres' })).toEqual(true);
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+
+    process.env.CUBEJS_DB_CLICKHOUSE_USE_COLLATION = 'false';
+    process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_USE_COLLATION = 'false';
+    process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_USE_COLLATION = 'false';
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'default' })).toEqual(false);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'postgres' })).toEqual(false);
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+
+    process.env.CUBEJS_DB_CLICKHOUSE_USE_COLLATION = 'wrong';
+    process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_USE_COLLATION = 'wrong';
+    process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_USE_COLLATION = 'wrong';
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'default' })).toThrow(
+      'The CUBEJS_DB_CLICKHOUSE_USE_COLLATION must be either \'true\' or \'false\'.'
+    );
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'postgres' })).toThrow(
+      'The CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_USE_COLLATION must be either \'true\' or \'false\'.'
+    );
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+
+    delete process.env.CUBEJS_DB_CLICKHOUSE_USE_COLLATION;
+    delete process.env.CUBEJS_DS_POSTGRES_DB_CLICKHOUSE_USE_COLLATION;
+    delete process.env.CUBEJS_DS_WRONG_DB_CLICKHOUSE_USE_COLLATION;
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'default' })).toEqual(true);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'postgres' })).toEqual(true);
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'wrong' })).toThrow(
+      'The wrong data source is missing in the declared CUBEJS_DATASOURCES.'
+    );
+  });
+
   test('getEnv("elasticApiId")', () => {
     process.env.CUBEJS_DB_ELASTIC_APIKEY_ID = 'default1';
     process.env.CUBEJS_DS_POSTGRES_DB_ELASTIC_APIKEY_ID = 'postgres1';

packages/cubejs-backend-shared/test/db_env_single.test.ts

@@ -975,6 +975,51 @@ describe('Single datasources', () => {
     expect(getEnv('clickhouseReadOnly', { dataSource: 'wrong' })).toBeUndefined();
   });
 
+  test('getEnv("clickhouseSortCollation")', () => {
+    process.env.CUBEJS_DB_CLICKHOUSE_SORT_COLLATION = 'default1';
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'default' })).toEqual('default1');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'postgres' })).toEqual('default1');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'wrong' })).toEqual('default1');
+
+    process.env.CUBEJS_DB_CLICKHOUSE_SORT_COLLATION = 'default2';
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'default' })).toEqual('default2');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'postgres' })).toEqual('default2');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'wrong' })).toEqual('default2');
+
+    delete process.env.CUBEJS_DB_CLICKHOUSE_SORT_COLLATION;
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'default' })).toEqual('en');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'postgres' })).toEqual('en');
+    expect(getEnv('clickhouseSortCollation', { dataSource: 'wrong' })).toEqual('en');
+  });
+
+  test('getEnv("clickhouseUseCollation")', () => {
+    process.env.CUBEJS_DB_CLICKHOUSE_USE_COLLATION = 'true';
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'default' })).toEqual(true);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'postgres' })).toEqual(true);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'wrong' })).toEqual(true);
+
+    process.env.CUBEJS_DB_CLICKHOUSE_USE_COLLATION = 'false';
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'default' })).toEqual(false);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'postgres' })).toEqual(false);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'wrong' })).toEqual(false);
+
+    process.env.CUBEJS_DB_CLICKHOUSE_USE_COLLATION = 'wrong';
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'default' })).toThrow(
+      'The CUBEJS_DB_CLICKHOUSE_USE_COLLATION must be either \'true\' or \'false\'.'
+    );
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'postgres' })).toThrow(
+      'The CUBEJS_DB_CLICKHOUSE_USE_COLLATION must be either \'true\' or \'false\'.'
+    );
+    expect(() => getEnv('clickhouseUseCollation', { dataSource: 'wrong' })).toThrow(
+      'The CUBEJS_DB_CLICKHOUSE_USE_COLLATION must be either \'true\' or \'false\'.'
+    );
+
+    delete process.env.CUBEJS_DB_CLICKHOUSE_USE_COLLATION;
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'default' })).toEqual(true);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'postgres' })).toEqual(true);
+    expect(getEnv('clickhouseUseCollation', { dataSource: 'wrong' })).toEqual(true);
+  });
+
   test('getEnv("elasticApiId")', () => {
     process.env.CUBEJS_DB_ELASTIC_APIKEY_ID = 'default1';
     expect(getEnv('elasticApiId', { dataSource: 'default' })).toEqual('default1');

packages/cubejs-schema-compiler/src/adapter/ClickHouseQuery.ts

@@ -1,4 +1,6 @@
-import { parseSqlInterval } from '@cubejs-backend/shared';
+import R from 'ramda';
+
+import { getEnv, parseSqlInterval } from '@cubejs-backend/shared';
 import { BaseQuery } from './BaseQuery';
 import { BaseFilter } from './BaseFilter';
 import { UserError } from '../compiler/UserError';
@@ -18,7 +20,7 @@ class ClickHouseFilter extends BaseFilter {
   public likeIgnoreCase(column, not, param, type) {
     const p = (!type || type === 'contains' || type === 'ends') ? '%' : '';
     const s = (!type || type === 'contains' || type === 'starts') ? '%' : '';
-    return `lower(${column}) ${not ? 'NOT' : ''} LIKE CONCAT('${p}', lower(${this.allocateParam(param)}), '${s}')`;
+    return `${column} ${not ? 'NOT' : ''} ILIKE CONCAT('${p}', ${this.allocateParam(param)}, '${s}')`;
   }
 
   public castParameter() {
@@ -123,7 +125,7 @@ export class ClickHouseQuery extends BaseQuery {
       .join(' AND ');
   }
 
-  public getFieldAlias(id) {
+  public getField(id) {
     const equalIgnoreCase = (a, b) => (
       typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
     );
@@ -134,16 +136,34 @@ export class ClickHouseQuery extends BaseQuery {
       d => equalIgnoreCase(d.dimension, id),
     );
 
+    if (!field) {
+      field = this.measures.find(
+        d => equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id),
+      );
+    }
+
+    return field;
+  }
+
+  public getFieldAlias(id) {
+    const field = this.getField(id);
+
     if (field) {
       return field.aliasName();
     }
 
-    field = this.measures.find(
-      d => equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id),
-    );
+    return null;
+  }
+
+  public getFieldType(hash) {
+    if (!hash || !hash.id) {
+      return null;
+    }
+
+    const field = this.getField(hash.id);
 
     if (field) {
-      return field.aliasName();
+      return field.definition().type;
     }
 
     return null;
@@ -168,6 +188,43 @@ export class ClickHouseQuery extends BaseQuery {
     return `${fieldAlias} ${direction}`;
   }
 
+  public getCollation() {
+    const useCollation = getEnv('clickhouseUseCollation', { dataSource: this.dataSource });
+    if (useCollation) {
+      return getEnv('clickhouseSortCollation', { dataSource: this.dataSource });
+    }
+    return null;
+  }
+
+  public override orderBy() {
+    //
+    // ClickHouse orders string by bytes, so we need to use COLLATE 'en' to order by string
+    //
+    if (R.isEmpty(this.order)) {
+      return '';
+    }
+
+    const collation = this.getCollation();
+
+    const orderByString = R.pipe(
+      R.map((order) => {
+        let orderString = this.orderHashToString(order);
+        if (collation && this.getFieldType(order) === 'string') {
+          orderString = `${orderString} COLLATE '${collation}'`;
+        }
+        return orderString;
+      }),
+      R.reject(R.isNil),
+      R.join(', ')
+    )(this.order);
+
+    if (!orderByString) {
+      return '';
+    }
+
+    return ` ORDER BY ${orderByString}`;
+  }
+
   public groupByClause() {
     if (this.ungrouped) {
       return '';
@@ -281,6 +338,22 @@ export class ClickHouseQuery extends BaseQuery {
     // ClickHouse intervals have a distinct type for each granularity
     delete templates.types.interval;
     delete templates.types.binary;
+
+    const collation = this.getCollation();
+
+    if (collation) {
+      templates.expressions.sort = `${templates.expressions.sort}{% if data_type and data_type == 'string' %} COLLATE '${collation}'{% endif %}`;
+      templates.expressions.order_by = `${templates.expressions.order_by}{% if data_type and data_type == 'string' %} COLLATE '${collation}'{% endif %}`;
+
+      const oldOrderBy = '{% if order_by %}\nORDER BY {{ order_by | map(attribute=\'expr\') | join(\', \') }}{% endif %}';
+
+      const newOrderBy =
+        '{% if order_by %}\nORDER BY {% for item in order_by %}{{ item.expr }}' +
+        `{%- if item.data_type and item.data_type == 'string' %} COLLATE '${collation}'{% endif %}` +
+        '{%- if not loop.last %}, {% endif %}{% endfor %}{% endif %}';
+
+      templates.statements.select = templates.statements.select.replace(oldOrderBy, newOrderBy);
+    }
     return templates;
   }
 }

packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts

@@ -65,7 +65,8 @@ export class ClickHouseDbRunner extends BaseDbRunner {
       (3, 300, '2017-01-05 16:00:00', '2017-01-19 16:00:00', 2, 'google', 120.120, 70.60),
       (4, 400, '2017-01-06 16:00:00', '2017-01-24 16:00:00', 2, null, 120.120, 10.60),
       (5, 500, '2017-01-06 16:00:00', '2017-01-24 16:00:00', 2, null, 120.120, 58.10),
-      (6, 500, '2016-09-06 16:00:00', '2016-09-06 16:00:00', 2, null, 120.120, 58.10)
+      (6, 500, '2016-09-06 16:00:00', '2016-09-06 16:00:00', 2, null, 120.120, 58.10),
+      (7, 300, '2017-01-07 16:00:00', '2017-01-25 16:00:00', 2, 'Gork', 120.120, 59.60)
     ` });
 
     await clickHouse.command({ query: `