feat(cubesql): Ungrouped SQL push down (#7102)

* feat(cubesql): Ungrouped SQL push down

* Match members instead of columns for push down

* Fix tests

* Dimension push down rules

* Dimension push down rules

* Move view evaluation to Cube Symbols so it's accessible in member expressions

* Use evaluateSymbolSql for member expressions to capture it's dependencies and allow full key query aggregate queries

* Fix warnings and add TODOs

* Add more TODOs

Author: paveltiunov

packages/cubejs-api-gateway/src/gateway.ts

@@ -49,7 +49,7 @@ import {
 } from './types/auth';
 import {
   Query,
-  NormalizedQuery,
+  NormalizedQuery, MemberExpression,
 } from './types/query';
 import {
   UserBackgroundContext,
@@ -89,6 +89,8 @@ import {
   transformPreAggregations,
 } from './helpers/transformMetaExtended';
 
+const memberExpressionRegex = /^([a-zA-Z0-9_]+).([a-zA-Z0-9_]+):\(([a-zA-Z0-9_,]+)\):(.*)$/;
+
 /**
  * API gateway server class.
  */
@@ -1170,18 +1172,23 @@ class ApiGateway {
     return [queryType, normalizedQueries];
   }
 
-  public async sql({ query, context, res, memberToAlias, exportAnnotatedSql }: QueryRequest) {
+  public async sql({ query, context, res, memberToAlias, exportAnnotatedSql, memberExpressions, expressionParams }: QueryRequest) {
     const requestStarted = new Date();
 
     try {
       await this.assertApiScope('data', context.securityContext);
 
       query = this.parseQueryParam(query);
+
+      if (memberExpressions) {
+        query = this.parseMemberExpressionsInQueries(query);
+      }
+
       const [queryType, normalizedQueries] = await this.getNormalizedQueries(query, context);
 
       const sqlQueries = await Promise.all<any>(
         normalizedQueries.map(async (normalizedQuery) => (await this.getCompilerApi(context)).getSql(
-          this.coerceForSqlQuery({ ...normalizedQuery, memberToAlias }, context),
+          this.coerceForSqlQuery({ ...normalizedQuery, memberToAlias, expressionParams }, context),
           {
             includeDebugInfo: getEnv('devMode') || context.signedWithPlaygroundAuthSecret,
             exportAnnotatedSql,
@@ -1204,6 +1211,39 @@ class ApiGateway {
     }
   }
 
+  private parseMemberExpressionsInQueries(query: Record<string, any> | Record<string, any>[]): Query | Query[] {
+    if (Array.isArray(query)) {
+      return query.map(q => this.parseMemberExpressionsInQuery(<Query>q));
+    } else {
+      return this.parseMemberExpressionsInQuery(<Query>query);
+    }
+  }
+
+  private parseMemberExpressionsInQuery(query: Query): Query {
+    return {
+      ...query,
+      measures: (query.measures || []).map(m => (typeof m === 'string' ? this.parseMemberExpression(m) : m)),
+      dimensions: (query.dimensions || []).map(m => (typeof m === 'string' ? this.parseMemberExpression(m) : m)),
+    };
+  }
+
+  private parseMemberExpression(memberExpression: string): string | MemberExpression {
+    const match = memberExpression.match(memberExpressionRegex);
+    if (match) {
+      const args = match[3].split(',');
+      args.push(`return \`${match[4]}\``);
+      return {
+        cubeName: match[1],
+        name: match[2],
+        expressionName: match[2],
+        expression: Function.constructor.apply(null, args),
+        definition: memberExpression,
+      };
+    } else {
+      return memberExpression;
+    }
+  }
+
   public async sqlGenerators({ context, res }: { context: RequestContext, res: ResponseResultFn }) {
     const requestStarted = new Date();
 
@@ -1651,6 +1691,8 @@ class ApiGateway {
       query = this.parseQueryParam(request.query);
       let resType: ResultType = ResultType.DEFAULT;
 
+      query = this.parseMemberExpressionsInQueries(query);
+
       if (!Array.isArray(query) && query.responseFormat) {
         resType = query.responseFormat;
       }

packages/cubejs-api-gateway/src/helpers/prepareAnnotation.ts

@@ -9,6 +9,7 @@ import R from 'ramda';
 import { MetaConfig, MetaConfigMap, toConfigMap } from './toConfigMap';
 import { MemberType } from '../types/strings';
 import { MemberType as MemberTypeEnum } from '../types/enums';
+import { MemberExpression } from '../types/query';
 
 /**
  * Annotation item for cube's member.
@@ -30,16 +31,16 @@ type ConfigItem = {
 const annotation = (
   configMap: MetaConfigMap,
   memberType: MemberType,
-) => (member: string): undefined | [string, ConfigItem] => {
-  const [cubeName, fieldName] = member.split('.');
+) => (member: string | MemberExpression): undefined | [string, ConfigItem] => {
+  const [cubeName, fieldName] = (<MemberExpression>member).expression ? [(<MemberExpression>member).cubeName, (<MemberExpression>member).name] : (<string>member).split('.');
   const memberWithoutGranularity = [cubeName, fieldName].join('.');
   const config: ConfigItem = configMap[cubeName][memberType]
     .find(m => m.name === memberWithoutGranularity);
 
   if (!config) {
     return undefined;
   }
-  return [member, {
+  return [typeof member === 'string' ? member : memberWithoutGranularity, {
     title: config.title,
     shortTitle: config.shortTitle,
     description: config.description,

packages/cubejs-api-gateway/src/query.js

@@ -40,6 +40,13 @@ const getPivotQuery = (queryType, queries) => {
 
 const id = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$/);
 const dimensionWithTime = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+(\.(second|minute|hour|day|week|month|year))?$/);
+const memberExpression = Joi.object().keys({
+  expression: Joi.func().required(),
+  cubeName: Joi.string().required(),
+  name: Joi.string().required(),
+  expressionName: Joi.string(),
+  definition: Joi.string(),
+});
 
 const operators = [
   'equals',
@@ -79,8 +86,9 @@ const oneCondition = Joi.object().keys({
 }).xor('or', 'and');
 
 const querySchema = Joi.object().keys({
-  measures: Joi.array().items(id),
-  dimensions: Joi.array().items(dimensionWithTime),
+  // TODO add member expression alternatives only for SQL API queries?
+  measures: Joi.array().items(Joi.alternatives(id, memberExpression)),
+  dimensions: Joi.array().items(Joi.alternatives(dimensionWithTime, memberExpression)),
   filters: Joi.array().items(oneFilter, oneCondition),
   timeDimensions: Joi.array().items(Joi.object().keys({
     dimension: id.required(),
@@ -176,7 +184,7 @@ const normalizeQuery = (query, persistent) => {
     );
   }
 
-  const regularToTimeDimension = (query.dimensions || []).filter(d => d.split('.').length === 3).map(d => ({
+  const regularToTimeDimension = (query.dimensions || []).filter(d => typeof d === 'string' && d.split('.').length === 3).map(d => ({
     dimension: d.split('.').slice(0, 2).join('.'),
     granularity: d.split('.')[2]
   }));
@@ -207,7 +215,7 @@ const normalizeQuery = (query, persistent) => {
     timezone,
     order: normalizeQueryOrder(query.order),
     filters: normalizeQueryFilters(query.filters || []),
-    dimensions: (query.dimensions || []).filter(d => d.split('.').length !== 3),
+    dimensions: (query.dimensions || []).filter(d => typeof d !== 'string' || d.split('.').length !== 3),
     timeDimensions: (query.timeDimensions || []).map(td => {
       let dateRange;
 

packages/cubejs-api-gateway/src/sql-server.ts

@@ -136,7 +136,7 @@ export class SQLServer {
           }
         });
       },
-      sql: async ({ request, session, query, memberToAlias }) => {
+      sql: async ({ request, session, query, memberToAlias, expressionParams }) => {
         const context = await contextByRequest(request, session);
 
         // eslint-disable-next-line no-async-promise-executor
@@ -145,7 +145,9 @@ export class SQLServer {
             await this.apiGateway.sql({
               query,
               memberToAlias,
+              expressionParams,
               exportAnnotatedSql: true,
+              memberExpressions: true,
               queryType: 'multi',
               context,
               res: (message) => {

packages/cubejs-api-gateway/src/types/query.ts

@@ -39,6 +39,14 @@ type LogicalOrFilter = {
   or: (QueryFilter | LogicalAndFilter)[]
 };
 
+type MemberExpression = {
+  expression: Function;
+  cubeName: string;
+  name: string;
+  expressionName: string;
+  definition: string;
+};
+
 /**
  * Query datetime dimention interface.
  */
@@ -52,8 +60,8 @@ interface QueryTimeDimension {
  * Incoming network query data type.
  */
 interface Query {
-  measures: Member[];
-  dimensions?: (Member | TimeMember)[];
+  measures: (Member | MemberExpression)[];
+  dimensions?: (Member | TimeMember | MemberExpression)[];
   filters?: (QueryFilter | LogicalAndFilter | LogicalOrFilter)[];
   timeDimensions?: QueryTimeDimension[];
   segments?: Member[];
@@ -92,4 +100,5 @@ export {
   Query,
   NormalizedQueryFilter,
   NormalizedQuery,
+  MemberExpression,
 };

packages/cubejs-api-gateway/src/types/request.ts

@@ -118,7 +118,9 @@ type QueryRequest = BaseRequest & {
   apiType?: ApiType;
   resType?: ResultType
   memberToAlias?: Record<string, string>;
+  expressionParams?: string[];
   exportAnnotatedSql?: boolean;
+  memberExpressions?: boolean;
 };
 
 type SqlApiRequest = BaseRequest & {

packages/cubejs-backend-native/js/index.ts

@@ -48,6 +48,7 @@ export interface SqlPayload {
     session: SessionContext,
     query: any,
     memberToAlias: Record<string, string>,
+    expressionParams: string[],
 }
 
 export interface SqlApiLoadPayload {

packages/cubejs-backend-native/src/transport.rs

@@ -68,6 +68,8 @@ struct LoadRequest {
     session: SessionContext,
     #[serde(rename = "memberToAlias", skip_serializing_if = "Option::is_none")]
     member_to_alias: Option<HashMap<String, String>>,
+    #[serde(rename = "expressionParams", skip_serializing_if = "Option::is_none")]
+    expression_params: Option<Vec<Option<String>>>,
     streaming: bool,
 }
 
@@ -176,6 +178,7 @@ impl TransportService for NodeBridgeTransport {
         ctx: AuthContextRef,
         meta: LoadRequestMeta,
         member_to_alias: Option<HashMap<String, String>>,
+        expression_params: Option<Vec<Option<String>>>,
     ) -> Result<SqlResponse, CubeError> {
         let native_auth = ctx
             .as_any()
@@ -196,6 +199,7 @@ impl TransportService for NodeBridgeTransport {
             },
             sql_query: None,
             member_to_alias,
+            expression_params,
             streaming: false,
         })?;
 
@@ -269,6 +273,7 @@ impl TransportService for NodeBridgeTransport {
                 },
                 sql_query: sql_query.clone().map(|q| (q.sql, q.values)),
                 member_to_alias: None,
+                expression_params: None,
                 streaming: false,
             })?;
 
@@ -345,6 +350,7 @@ impl TransportService for NodeBridgeTransport {
                     superuser: native_auth.superuser,
                 },
                 member_to_alias: None,
+                expression_params: None,
                 streaming: true,
             })?;
 

packages/cubejs-schema-compiler/src/adapter/BaseDimension.js

@@ -1,6 +1,12 @@
 export class BaseDimension {
   constructor(query, dimension) {
     this.query = query;
+    if (dimension && dimension.expression) {
+      this.expression = dimension.expression;
+      this.expressionCubeName = dimension.cubeName;
+      this.expressionName = dimension.expressionName || `${dimension.cubeName}.${dimension.name}`;
+      this.isMemberExpression = !!dimension.definition;
+    }
     this.dimension = dimension;
   }
 
@@ -17,6 +23,9 @@ export class BaseDimension {
   }
 
   dimensionSql() {
+    if (this.expression) {
+      return this.query.evaluateSymbolSql(this.expressionCubeName, this.expressionName, this.definition(), 'dimension');
+    }
     if (this.query.cubeEvaluator.isSegment(this.dimension)) {
       return this.query.wrapSegmentForDimensionSelect(this.query.dimensionSql(this));
     }
@@ -32,6 +41,9 @@ export class BaseDimension {
   }
 
   cube() {
+    if (this.expression) {
+      return this.query.cubeEvaluator.cubeFromPath(this.expressionCubeName);
+    }
     return this.query.cubeEvaluator.cubeFromPath(this.dimension);
   }
 
@@ -43,6 +55,13 @@ export class BaseDimension {
   }
 
   definition() {
+    if (this.expression) {
+      return {
+        sql: this.expression,
+        // TODO use actual dimension type even though it isn't used right now
+        type: 'number'
+      };
+    }
     return this.dimensionDefinition();
   }
 
@@ -52,6 +71,9 @@ export class BaseDimension {
   }
 
   unescapedAliasName() {
+    if (this.expression) {
+      return this.query.aliasName(this.expressionName);
+    }
     return this.query.aliasName(this.dimension);
   }
 
@@ -60,6 +82,9 @@ export class BaseDimension {
   }
 
   path() {
+    if (this.expression) {
+      return null;
+    }
     if (this.query.cubeEvaluator.isSegment(this.dimension)) {
       return this.query.cubeEvaluator.parsePath('segments', this.dimension);
     }

packages/cubejs-schema-compiler/src/adapter/BaseMeasure.js

@@ -6,7 +6,8 @@ export class BaseMeasure {
     if (measure.expression) {
       this.expression = measure.expression;
       this.expressionCubeName = measure.cubeName;
-      this.expressionName = `${measure.cubeName}.${measure.name}`;
+      this.expressionName = measure.expressionName || `${measure.cubeName}.${measure.name}`;
+      this.isMemberExpression = !!measure.definition;
     }
     this.measure = measure;
   }
@@ -38,7 +39,7 @@ export class BaseMeasure {
 
   measureSql() {
     if (this.expression) {
-      return this.query.evaluateSql(this.expressionCubeName, this.expression);
+      return this.query.evaluateSymbolSql(this.expressionCubeName, this.expressionName, this.definition(), 'measure');
     }
     return this.query.measureSql(this);
   }
@@ -58,6 +59,7 @@ export class BaseMeasure {
     if (this.expression) {
       return {
         sql: this.expression,
+        // TODO use actual measure type even though it isn't used right now
         type: 'number'
       };
     }