chore(sqlplanner): Sql planner another schema refactoring (#9038)

* template engine

* in work

* in work

* multi stage and running total pass tests

* update

* update

* fix(cubeplanner): Fixes to pass the tests

* some fixes

* views support

* evaluation_node refactoring

* in work

* in work

* in work

* update

* update

Author: waralexrom

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -11,7 +11,7 @@ import cronParser from 'cron-parser';
 
 import moment from 'moment-timezone';
 import inflection from 'inflection';
-import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv } from '@cubejs-backend/shared';
+import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv, timeSeries as timeSeriesBase } from '@cubejs-backend/shared';
 
 import {
   buildSqlAndParams as nativeBuildSqlAndParams,
@@ -606,6 +606,11 @@ export class BaseQuery {
   }
 
   buildSqlAndParamsRust(exportAnnotatedSql) {
+    const order = this.options.order && R.pipe(
+      R.map((hash) => ((!hash || !hash.id) ? null : hash)),
+      R.reject(R.isNil),
+    )(this.options.order);
+
     const queryParams = {
       measures: this.options.measures,
       dimensions: this.options.dimensions,
@@ -614,12 +619,13 @@ export class BaseQuery {
       joinRoot: this.join.root,
       joinGraph: this.joinGraph,
       cubeEvaluator: this.cubeEvaluator,
-      order: this.options.order,
+      order,
       filters: this.options.filters,
       limit: this.options.limit ? this.options.limit.toString() : null,
       rowLimit: this.options.rowLimit ? this.options.rowLimit.toString() : null,
       offset: this.options.offset ? this.options.offset.toString() : null,
       baseTools: this,
+      ungrouped: this.options.ungrouped
 
     };
     const res = nativeBuildSqlAndParams(queryParams);
@@ -628,6 +634,21 @@ export class BaseQuery {
     return res;
   }
 
+  allCubeMembers(path) {
+    const fromPath = this.cubeEvaluator.cubeFromPath(path);
+
+    return Object.keys(fromPath.measures).concat(Object.keys(fromPath.dimensions));
+  }
+
+  getAllocatedParams() {
+    return this.paramAllocator.getParams();
+  }
+
+  // FIXME helper for native generator, maybe should be moved entire to rust
+  generateTimeSeries(granularity, dateRange) {
+    return timeSeriesBase(granularity, dateRange);
+  }
+
   get shouldReuseParams() {
     return false;
   }
@@ -3236,7 +3257,16 @@ export class BaseQuery {
           '{% if offset is not none %}\nOFFSET {{ offset }}{% endif %}',
         group_by_exprs: '{{ group_by | map(attribute=\'index\') | join(\', \') }}',
         join: '{{ join_type }} JOIN {{ source }} ON {{ condition }}',
-        cte: '{{ alias }} AS ({{ query | indent(2, true) }})'
+        cte: '{{ alias }} AS ({{ query | indent(2, true) }})',
+        time_series_select: 'SELECT date_from::timestamp AS "date_from",\n' +
+        'date_to::timestamp AS "date_to" \n' +
+        'FROM(\n' +
+        '    VALUES ' +
+        '{% for time_item in seria  %}' +
+        '(\'{{ time_item | join(\'\\\', \\\'\') }}\')' +
+        '{% if not loop.last %}, {% endif %}' +
+        '{% endfor %}' +
+        ') AS dates (date_from, date_to)'
       },
       expressions: {
         column_reference: '{% if table_name %}{{ table_name }}.{% endif %}{{ name }}',
@@ -3257,6 +3287,8 @@ export class BaseQuery {
         cube: 'CUBE({{ exprs_concat }})',
         negative: '-({{ expr }})',
         not: 'NOT ({{ expr }})',
+        add_interval: '{{ date }} + interval \'{{ interval }}\'',
+        sub_interval: '{{ date }} - interval \'{{ interval }}\'',
         true: 'TRUE',
         false: 'FALSE',
         like: '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}',
@@ -3276,7 +3308,8 @@ export class BaseQuery {
         gte: '{{ column }} >= {{ param }}',
         lt: '{{ column }} < {{ param }}',
         lte: '{{ column }} <= {{ param }}',
-        always_true: '1 == 1'
+        like_pattern: '{% if start_wild %}\'%\' || {% endif %}{{ value }}{% if end_wild %}|| \'%\'{% endif %}',
+        always_true: '1 = 1'
 
       },
       quotes: {

packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts

@@ -51,6 +51,10 @@ export class ParamAllocator {
     return `$${paramIndex}$`;
   }
 
+  public getParams() {
+    return this.params;
+  }
+
   // eslint-disable-next-line no-unused-vars
   protected paramPlaceHolder(paramIndex) {
     return '?';

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -106,7 +106,7 @@ export class CubeSymbols {
       },
       set segments(v) {
         // Dont allow to modify
-      }
+      },
     }, cubeDefinition);
 
     if (cubeDefinition.extends) {

packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts

@@ -5,7 +5,7 @@ import type { StartedTestContainer } from 'testcontainers';
 import { format as formatSql } from 'sqlstring';
 import { v4 as uuidv4 } from 'uuid';
 import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery';
-import { BaseDbRunner } from "../utils/BaseDbRunner";
+import { BaseDbRunner } from '../utils/BaseDbRunner';
 
 process.env.TZ = 'GMT';
 

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts

@@ -453,7 +453,7 @@ describe('SQL Generation', () => {
     }
   });
 
-  it('having filter with operator OR', async () => {
+  it('having filter with operator OR 1', async () => {
     await compiler.compile();
 
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
@@ -648,7 +648,7 @@ describe('SQL Generation', () => {
     });
   });
 
-  it('where filter with operators OR & AND', async () => {
+  it('where filter with operators OR & AND 1', async () => {
     await compiler.compile();
 
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts

@@ -64,7 +64,7 @@ describe('SQL Generation', () => {
             offset: 'start'
           }
         },
-        revenueRolling3day: {
+        revenueRollingThreeDay: {
           type: 'sum',
           sql: 'amount',
           rollingWindow: {
@@ -780,7 +780,7 @@ describe('SQL Generation', () => {
 
   it('rolling month', async () => runQueryTest({
     measures: [
-      'visitors.revenueRolling3day'
+      'visitors.revenueRollingThreeDay'
     ],
     timeDimensions: [{
       dimension: 'visitors.created_at',
@@ -792,7 +792,7 @@ describe('SQL Generation', () => {
     }],
     timezone: 'America/Los_Angeles'
   }, [
-    { visitors__created_at_week: '2017-01-09T00:00:00.000Z', visitors__revenue_rolling3day: '900' }
+    { visitors__created_at_week: '2017-01-09T00:00:00.000Z', visitors__revenue_rolling_three_day: '900' }
   ]));
 
   it('rolling count', async () => runQueryTest({
@@ -1791,7 +1791,7 @@ describe('SQL Generation', () => {
   ]));
 
   it(
-    'contains filter',
+    'contains filter 1',
     () => runQueryTest({
       measures: [],
       dimensions: [
@@ -2583,7 +2583,7 @@ describe('SQL Generation', () => {
     ]
   ));
 
-  it('rank measure', async () => runQueryTest(
+  it('rank measure 1', async () => runQueryTest(
     {
       measures: ['visitors.revenue_rank'],
     },

packages/cubejs-schema-compiler/test/unit/base-query.test.ts

@@ -1596,7 +1596,7 @@ describe('SQL Generation', () => {
               sql: 'product_id',
               type: 'avg',
               filters: [
-                { sql: `{FILTER_PARAMS.Order.category.filter('category')}` }
+                { sql: '{FILTER_PARAMS.Order.category.filter(\'category\')}' }
               ]
             }
           ],
@@ -1613,7 +1613,7 @@ describe('SQL Generation', () => {
             },
             {
               name: 'proxied',
-              sql: `{FILTER_PARAMS.Order.type.filter("x => type = 'online'")}`,
+              sql: '{FILTER_PARAMS.Order.type.filter("x => type = \'online\'")}',
               type: 'boolean',
             }
           ]

packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts

@@ -217,8 +217,8 @@ describe('pre-aggregations', () => {
     console.log(JSON.stringify(preAggregationsDescription, null, 2));
 
     expect(preAggregationsDescription.length).toEqual(2);
-    expect(preAggregationsDescription[0].preAggregationId).toEqual("Orders.simple1");
-    expect(preAggregationsDescription[1].preAggregationId).toEqual("Orders.simple2");
+    expect(preAggregationsDescription[0].preAggregationId).toEqual('Orders.simple1');
+    expect(preAggregationsDescription[1].preAggregationId).toEqual('Orders.simple2');
   });
 
   // @link https://github.com/cube-js/cube/issues/6623

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs

@@ -60,6 +60,7 @@ pub struct BaseQueryOptionsStatic {
     #[serde(rename = "rowLimit")]
     pub row_limit: Option<String>,
     pub offset: Option<String>,
+    pub ungrouped: Option<bool>,
 }
 
 #[nativebridge::native_bridge(BaseQueryOptionsStatic)]

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs

@@ -38,4 +38,11 @@ pub trait BaseTools {
     fn filter_group_function(&self) -> Result<Rc<dyn FilterGroup>, CubeError>;
     fn timestamp_precision(&self) -> Result<u32, CubeError>;
     fn in_db_time_zone(&self, date: String) -> Result<String, CubeError>;
+    fn generate_time_series(
+        &self,
+        granularity: String,
+        date_range: Vec<String>,
+    ) -> Result<Vec<Vec<String>>, CubeError>;
+    fn get_allocated_params(&self) -> Result<Vec<String>, CubeError>;
+    fn all_cube_members(&self, path: String) -> Result<Vec<String>, CubeError>;
 }