chore(sqlplanner): Sql planner another schema refactoring

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -11,7 +11,7 @@ import cronParser from 'cron-parser';
 
 import moment from 'moment-timezone';
 import inflection from 'inflection';
-import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv } from '@cubejs-backend/shared';
+import { FROM_PARTITION_RANGE, inDbTimeZone, MAX_SOURCE_ROW_LIMIT, QueryAlias, getEnv, timeSeries as timeSeriesBase } from '@cubejs-backend/shared';
 
 import {
   buildSqlAndParams as nativeBuildSqlAndParams,
@@ -606,6 +606,11 @@ export class BaseQuery {
   }
 
   buildSqlAndParamsRust(exportAnnotatedSql) {
+    const order = this.options.order && R.pipe(
+      R.map((hash) => ((!hash || !hash.id) ? null : hash)),
+      R.reject(R.isNil),
+    )(this.options.order);
+
     const queryParams = {
       measures: this.options.measures,
       dimensions: this.options.dimensions,
@@ -614,12 +619,13 @@ export class BaseQuery {
       joinRoot: this.join.root,
       joinGraph: this.joinGraph,
       cubeEvaluator: this.cubeEvaluator,
-      order: this.options.order,
+      order,
       filters: this.options.filters,
       limit: this.options.limit ? this.options.limit.toString() : null,
       rowLimit: this.options.rowLimit ? this.options.rowLimit.toString() : null,
       offset: this.options.offset ? this.options.offset.toString() : null,
       baseTools: this,
+      ungrouped: this.options.ungrouped
 
     };
     const res = nativeBuildSqlAndParams(queryParams);
@@ -628,6 +634,21 @@ export class BaseQuery {
     return res;
   }
 
+  allCubeMembers(path) {
+    const fromPath = this.cubeEvaluator.cubeFromPath(path);
+
+    return Object.keys(fromPath.measures).concat(Object.keys(fromPath.dimensions));
+  }
+
+  getAllocatedParams() {
+    return this.paramAllocator.getParams();
+  }
+
+  // FIXME helper for native generator, maybe should be moved entire to rust
+  generateTimeSeries(granularity, dateRange) {
+    return timeSeriesBase(granularity, dateRange);
+  }
+
   get shouldReuseParams() {
     return false;
   }
@@ -3232,7 +3253,16 @@ export class BaseQuery {
           '{% if offset is not none %}\nOFFSET {{ offset }}{% endif %}',
         group_by_exprs: '{{ group_by | map(attribute=\'index\') | join(\', \') }}',
         join: '{{ join_type }} JOIN {{ source }} ON {{ condition }}',
-        cte: '{{ alias }} AS ({{ query | indent(2, true) }})'
+        cte: '{{ alias }} AS ({{ query | indent(2, true) }})',
+        time_series_select: 'SELECT date_from::timestamp AS "date_from",\n' +
+        'date_to::timestamp AS "date_to" \n' +
+        'FROM(\n' +
+        '    VALUES ' +
+        '{% for time_item in seria  %}' +
+        '(\'{{ time_item | join(\'\\\', \\\'\') }}\')' +
+        '{% if not loop.last %}, {% endif %}' +
+        '{% endfor %}' +
+        ') AS dates (date_from, date_to)'
       },
       expressions: {
         column_reference: '{% if table_name %}{{ table_name }}.{% endif %}{{ name }}',
@@ -3253,6 +3283,8 @@ export class BaseQuery {
         cube: 'CUBE({{ exprs_concat }})',
         negative: '-({{ expr }})',
         not: 'NOT ({{ expr }})',
+        add_interval: '{{ date }} + interval \'{{ interval }}\'',
+        sub_interval: '{{ date }} - interval \'{{ interval }}\'',
         true: 'TRUE',
         false: 'FALSE',
         like: '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}',
@@ -3272,7 +3304,8 @@ export class BaseQuery {
         gte: '{{ column }} >= {{ param }}',
         lt: '{{ column }} < {{ param }}',
         lte: '{{ column }} <= {{ param }}',
-        always_true: '1 == 1'
+        like_pattern: '{% if start_wild %}\'%\' || {% endif %}{{ value }}{% if end_wild %}|| \'%\'{% endif %}',
+        always_true: '1 = 1'
 
       },
       quotes: {

packages/cubejs-schema-compiler/src/adapter/ParamAllocator.ts

@@ -51,6 +51,10 @@ export class ParamAllocator {
     return `$${paramIndex}$`;
   }
 
+  public getParams() {
+    return this.params;
+  }
+
   // eslint-disable-next-line no-unused-vars
   protected paramPlaceHolder(paramIndex) {
     return '?';

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js

@@ -106,7 +106,7 @@ export class CubeSymbols {
       },
       set segments(v) {
         // Dont allow to modify
-      }
+      },
     }, cubeDefinition);
 
     if (cubeDefinition.extends) {

packages/cubejs-schema-compiler/test/integration/clickhouse/ClickHouseDbRunner.ts

@@ -5,7 +5,7 @@ import type { StartedTestContainer } from 'testcontainers';
 import { format as formatSql } from 'sqlstring';
 import { v4 as uuidv4 } from 'uuid';
 import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery';
-import { BaseDbRunner } from "../utils/BaseDbRunner";
+import { BaseDbRunner } from '../utils/BaseDbRunner';
 
 process.env.TZ = 'GMT';
 

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts

@@ -453,7 +453,7 @@ describe('SQL Generation', () => {
     }
   });
 
-  it('having filter with operator OR', async () => {
+  it('having filter with operator OR 1', async () => {
     await compiler.compile();
 
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
@@ -648,7 +648,7 @@ describe('SQL Generation', () => {
     });
   });
 
-  it('where filter with operators OR & AND', async () => {
+  it('where filter with operators OR & AND 1', async () => {
     await compiler.compile();
 
     const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts

@@ -64,7 +64,7 @@ describe('SQL Generation', () => {
             offset: 'start'
           }
         },
-        revenueRolling3day: {
+        revenueRollingThreeDay: {
           type: 'sum',
           sql: 'amount',
           rollingWindow: {
@@ -769,7 +769,7 @@ describe('SQL Generation', () => {
 
   it('rolling month', async () => runQueryTest({
     measures: [
-      'visitors.revenueRolling3day'
+      'visitors.revenueRollingThreeDay'
     ],
     timeDimensions: [{
       dimension: 'visitors.created_at',
@@ -781,7 +781,7 @@ describe('SQL Generation', () => {
     }],
     timezone: 'America/Los_Angeles'
   }, [
-    { visitors__created_at_week: '2017-01-09T00:00:00.000Z', visitors__revenue_rolling3day: '900' }
+    { visitors__created_at_week: '2017-01-09T00:00:00.000Z', visitors__revenue_rolling_three_day: '900' }
   ]));
 
   it('rolling count', async () => runQueryTest({
@@ -1721,7 +1721,7 @@ describe('SQL Generation', () => {
   ]));
 
   it(
-    'contains filter',
+    'contains filter 1',
     () => runQueryTest({
       measures: [],
       dimensions: [
@@ -2348,7 +2348,7 @@ describe('SQL Generation', () => {
     ]
   ));
 
-  it('rank measure', async () => runQueryTest(
+  it('rank measure 1', async () => runQueryTest(
     {
       measures: ['visitors.revenue_rank'],
     },

packages/cubejs-schema-compiler/test/unit/base-query.test.ts

@@ -1596,7 +1596,7 @@ describe('SQL Generation', () => {
               sql: 'product_id',
               type: 'avg',
               filters: [
-                { sql: `{FILTER_PARAMS.Order.category.filter('category')}` }
+                { sql: '{FILTER_PARAMS.Order.category.filter(\'category\')}' }
               ]
             }
           ],
@@ -1613,7 +1613,7 @@ describe('SQL Generation', () => {
             },
             {
               name: 'proxied',
-              sql: `{FILTER_PARAMS.Order.type.filter("x => type = 'online'")}`,
+              sql: '{FILTER_PARAMS.Order.type.filter("x => type = \'online\'")}',
               type: 'boolean',
             }
           ]

packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts

@@ -217,8 +217,8 @@ describe('pre-aggregations', () => {
     console.log(JSON.stringify(preAggregationsDescription, null, 2));
 
     expect(preAggregationsDescription.length).toEqual(2);
-    expect(preAggregationsDescription[0].preAggregationId).toEqual("Orders.simple1");
-    expect(preAggregationsDescription[1].preAggregationId).toEqual("Orders.simple2");
+    expect(preAggregationsDescription[0].preAggregationId).toEqual('Orders.simple1');
+    expect(preAggregationsDescription[1].preAggregationId).toEqual('Orders.simple2');
   });
 
   // @link https://github.com/cube-js/cube/issues/6623

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs

@@ -60,6 +60,7 @@ pub struct BaseQueryOptionsStatic {
     #[serde(rename = "rowLimit")]
     pub row_limit: Option<String>,
     pub offset: Option<String>,
+    pub ungrouped: Option<bool>,
 }
 
 #[nativebridge::native_bridge(BaseQueryOptionsStatic)]

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs

@@ -38,4 +38,11 @@ pub trait BaseTools {
     fn filter_group_function(&self) -> Result<Rc<dyn FilterGroup>, CubeError>;
     fn timestamp_precision(&self) -> Result<u32, CubeError>;
     fn in_db_time_zone(&self, date: String) -> Result<String, CubeError>;
+    fn generate_time_series(
+        &self,
+        granularity: String,
+        date_range: Vec<String>,
+    ) -> Result<Vec<Vec<String>>, CubeError>;
+    fn get_allocated_params(&self) -> Result<Vec<String>, CubeError>;
+    fn all_cube_members(&self, path: String) -> Result<Vec<String>, CubeError>;
 }

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs

@@ -21,6 +21,13 @@ pub struct TimeShiftReference {
     pub time_dimension: String,
 }
 
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct RollingWindow {
+    pub trailing: Option<String>,
+    pub leading: Option<String>,
+    pub offset: Option<String>,
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 pub struct MeasureDefinitionStatic {
     #[serde(rename = "type")]
@@ -37,6 +44,8 @@ pub struct MeasureDefinitionStatic {
     pub group_by_references: Option<Vec<String>>,
     #[serde(rename = "timeShiftReferences")]
     pub time_shift_references: Option<Vec<TimeShiftReference>>,
+    #[serde(rename = "rollingWindow")]
+    pub rolling_window: Option<RollingWindow>,
 }
 
 #[nativebridge::native_bridge(MeasureDefinitionStatic)]

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/memeber_sql.rs

@@ -19,7 +19,7 @@ use std::rc::Rc;
 pub struct MemberSqlStruct {
     pub sql_fn: Option<String>,
     pub to_string_fn: Option<String>,
-    pub properties: HashMap<String, String>,
+    pub properties: HashMap<String, MemberSqlArg>,
 }
 
 pub enum ContextSymbolArg {
@@ -64,6 +64,39 @@ impl<IT: InnerTypes> NativeSerialize<IT> for MemberSqlStruct {
     }
 }
 
+impl<IT: InnerTypes> NativeSerialize<IT> for MemberSqlArg {
+    fn to_native(
+        &self,
+        context_holder: NativeContextHolder<IT>,
+    ) -> Result<NativeObjectHandle<IT>, CubeError> {
+        let res = match self {
+            MemberSqlArg::String(s) => s.to_native(context_holder.clone()),
+            MemberSqlArg::Struct(s) => s.to_native(context_holder.clone()),
+            MemberSqlArg::ContextSymbol(symbol) => match symbol {
+                ContextSymbolArg::SecurityContext(context) => context
+                    .clone()
+                    .as_any()
+                    .downcast::<NativeSecurityContext<IT>>()
+                    .unwrap()
+                    .to_native(context_holder.clone()),
+                ContextSymbolArg::FilterParams(params) => params
+                    .clone()
+                    .as_any()
+                    .downcast::<NativeFilterParams<IT>>()
+                    .unwrap()
+                    .to_native(context_holder.clone()),
+                ContextSymbolArg::FilterGroup(group) => group
+                    .clone()
+                    .as_any()
+                    .downcast::<NativeFilterGroup<IT>>()
+                    .unwrap()
+                    .to_native(context_holder.clone()),
+            },
+        }?;
+        Ok(NativeObjectHandle::new(res.into_object()))
+    }
+}
+
 impl<IT: InnerTypes> NativeMemberSql<IT> {
     pub fn try_new(native_object: NativeObjectHandle<IT>) -> Result<Self, CubeError> {
         let args_names = native_object.to_function()?.args_names()?;
@@ -89,27 +122,7 @@ impl<IT: InnerTypes> MemberSql for NativeMemberSql<IT> {
         let context_holder = NativeContextHolder::<IT>::new(self.native_object.get_context());
         let native_args = args
             .into_iter()
-            .map(|a| match a {
-                MemberSqlArg::String(s) => s.to_native(context_holder.clone()),
-                MemberSqlArg::Struct(s) => s.to_native(context_holder.clone()),
-                MemberSqlArg::ContextSymbol(symbol) => match symbol {
-                    ContextSymbolArg::SecurityContext(context) => context
-                        .as_any()
-                        .downcast::<NativeSecurityContext<IT>>()
-                        .unwrap()
-                        .to_native(context_holder.clone()),
-                    ContextSymbolArg::FilterParams(params) => params
-                        .as_any()
-                        .downcast::<NativeFilterParams<IT>>()
-                        .unwrap()
-                        .to_native(context_holder.clone()),
-                    ContextSymbolArg::FilterGroup(group) => group
-                        .as_any()
-                        .downcast::<NativeFilterGroup<IT>>()
-                        .unwrap()
-                        .to_native(context_holder.clone()),
-                },
-            })
+            .map(|a| a.to_native(context_holder.clone()))
             .collect::<Result<Vec<_>, _>>()?;
 
         let res = self.native_object.to_function()?.call(native_args)?;