Merge branch 'tesseract-ci' of github.com:cube-js/cube into tesseract-ci

Author: KSDaemon

docs/pages/product/deployment/cloud/vpc/aws/private-link.mdx

@@ -21,7 +21,7 @@ for creating and managing an Endpoint Service.
 
 Cube Cloud needs to be added to the list of principals allowed to discover your Endpoint Service.
 To do so, please go to <Btn>AWS Console</Btn> -> <Btn>VPC</Btn> -> <Btn>Endpoint Services</Btn> -> <Btn>Your service</Btn> -> <Btn>Allow principals</Btn>
-and add `arn:aws:iam::331376342520` to the list.
+and add `arn:aws:iam::331376342520:root` to the list.
 
 ## Gathering required information
 
@@ -31,8 +31,8 @@ To request establishing a PrivateLink connection, please share the following inf
 - **Reference Name** for the record (such as "Snowflake-prod" or "clickhouse-dev")
 - **Ports**: a list of ports that will be accessed through this connection
 - **DNS Name** (optional): an internal DNS name of the upstream service in case SSL needs to be supported
-- **Dedicated Infrastructure Region:** VPC Peering requires Cube to be hosted in 
-    [dedicated infrastructure][dedicated-infra]. Please specify what region the Cube Cloud 
+- **Dedicated Infrastructure Region:** VPC Peering requires Cube to be hosted in
+    [dedicated infrastructure][dedicated-infra]. Please specify what region the Cube Cloud
     dedicated infrastructure should be hosted in.
 
 
@@ -52,4 +52,4 @@ supplied DNS Name or an AWS internal DNS name returned to you by the Cube team.
 
 [aws-docs-private-link]: https://aws.amazon.com/privatelink/
 [aws-docs-endpoint-service]: https://docs.aws.amazon.com/vpc/latest/privatelink/configure-endpoint-service.html
-[dedicated-infra]: /product/deployment/cloud/infrastructure#dedicated-infrastructure
+[dedicated-infra]: /product/deployment/cloud/infrastructure#dedicated-infrastructure

packages/cubejs-api-gateway/src/query.js

@@ -56,7 +56,8 @@ const evaluatedPatchMeasureExpression = parsedPatchMeasureExpression.keys({
 });
 
 const id = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$/);
-const idOrMemberExpressionName = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+$/);
+// It might be member name, td+granularity or member expression
+const idOrMemberExpressionName = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$/);
 const dimensionWithTime = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+(\.[a-zA-Z0-9_]+)?$/);
 const parsedMemberExpression = Joi.object().keys({
   expression: Joi.alternatives(

packages/cubejs-backend-shared/src/FileRepository.ts

@@ -5,6 +5,7 @@ export interface FileContent {
   fileName: string;
   content: string;
   readOnly?: boolean;
+  isModule?: boolean;
 }
 
 export interface SchemaFileRepository {
@@ -24,7 +25,7 @@ export class FileRepository implements SchemaFileRepository {
 
   protected async getFiles(dir: string, fileList: string[] = []): Promise<string[]> {
     let files: string[] = [];
-    
+
     try {
       const fullPath = path.join(this.localPath(), dir);
       await fs.ensureDir(fullPath);

packages/cubejs-backend-shared/src/time.ts

@@ -12,6 +12,67 @@ export type TimeSeriesOptions = {
 };
 type ParsedInterval = Partial<Record<unitOfTime.DurationConstructor, number>>;
 
+const GRANULARITY_LEVELS: Record<string, number> = {
+  second: 1,
+  minute: 2,
+  hour: 3,
+  day: 4,
+  week: 5,
+  month: 6,
+  quarter: 7,
+  year: 8,
+  MAX: 1000,
+};
+
+export type DimensionToCompareGranularity = {
+  dimension: string;
+  expressionName?: string;
+  granularityObj?: {
+    minGranularity(): string;
+  }
+};
+
+/**
+ * Actually dimensions type is (BaseDimension|BaseTimeDimension)[], but can not ref due to cyclic dependencies refs.
+ */
+export function findMinGranularityDimension(id: string, dimensions: DimensionToCompareGranularity[]): { index: number, dimension: DimensionToCompareGranularity | undefined } | null {
+  const equalIgnoreCase = (a: any, b: any): boolean => (
+    typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
+  );
+
+  let minGranularity = GRANULARITY_LEVELS.MAX;
+  let index = -1;
+  let minGranularityIndex = -1;
+  let field;
+  let minGranularityField;
+
+  dimensions.forEach((d, i) => {
+    if (equalIgnoreCase(d.dimension, id) || equalIgnoreCase(d.expressionName, id)) {
+      field = d;
+      index = i;
+
+      if ('granularityObj' in d && d.granularityObj) {
+        const gr = GRANULARITY_LEVELS[d.granularityObj?.minGranularity()];
+        if (gr < minGranularity) {
+          minGranularityIndex = i;
+          minGranularityField = d;
+          minGranularity = gr;
+        }
+      }
+    }
+  });
+
+  if (minGranularityIndex > -1) {
+    return { index: minGranularityIndex, dimension: minGranularityField };
+  }
+
+  if (index > -1) {
+    return { index, dimension: field };
+  }
+
+  return null;
+}
+
 export const TIME_SERIES: Record<string, (range: DateRange, timestampPrecision: number) => QueryDateRange[]> = {
   day: (range: DateRange, digits) => Array.from(range.snapTo('day').by('day'))
     .map(d => [d.format(`YYYY-MM-DDT00:00:00.${'0'.repeat(digits)}`), d.format(`YYYY-MM-DDT23:59:59.${'9'.repeat(digits)}`)]),

packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts

@@ -144,42 +144,19 @@ export class DatabricksQuery extends BaseQuery {
     return `\`${name}\``;
   }
 
-  public getFieldIndex(id: string) {
-    const dimension = this.dimensionsForSelect().find((d: any) => d.dimension === id);
-    if (dimension) {
-      return super.getFieldIndex(id);
+  public override getFieldIndex(id: string): string | number | null {
+    const idx = super.getFieldIndex(id);
+    if (idx !== null) {
+      return idx;
     }
+
     return this.escapeColumnName(this.aliasName(id, false));
   }
 
   public unixTimestampSql() {
     return 'unix_timestamp()';
   }
 
-  public orderHashToString(hash: any) {
-    if (!hash || !hash.id) {
-      return null;
-    }
-
-    const fieldIndex = this.getFieldIndex(hash.id);
-    if (fieldIndex === null) {
-      return null;
-    }
-
-    const dimensionsForSelect = this.dimensionsForSelect();
-    const dimensionColumns = R.flatten(
-      dimensionsForSelect.map((s: any) => s.selectColumns() && s.aliasName())
-    )
-      .filter(s => !!s);
-
-    if (dimensionColumns.length) {
-      const direction = hash.desc ? 'DESC' : 'ASC';
-      return `${fieldIndex} ${direction}`;
-    }
-
-    return null;
-  }
-
   public defaultRefreshKeyRenewalThreshold() {
     return 120;
   }

packages/cubejs-questdb-driver/src/QuestQuery.ts

@@ -1,4 +1,8 @@
-import { BaseFilter, BaseQuery, ParamAllocator } from '@cubejs-backend/schema-compiler';
+import {
+  BaseFilter,
+  BaseQuery,
+  ParamAllocator
+} from '@cubejs-backend/schema-compiler';
 
 const GRANULARITY_TO_INTERVAL: Record<string, string> = {
   second: 's',
@@ -109,7 +113,7 @@ export class QuestQuery extends BaseQuery {
     }
   }
 
-  public orderHashToString(hash: any): string | null {
+  public orderHashToString(hash: { id: string, desc: boolean }): string | null {
     // QuestDB has partial support for order by index column, so map these to the alias names.
     // So, instead of:
     // SELECT col_a as "a", col_b as "b" FROM tab ORDER BY 2 ASC
@@ -131,32 +135,6 @@ export class QuestQuery extends BaseQuery {
     return `${fieldAlias} ${direction}`;
   }
 
-  private getFieldAlias(id: string): string | null {
-    const equalIgnoreCase = (a: any, b: any) => (
-      typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
-    );
-
-    let field;
-
-    field = this.dimensionsForSelect().find(
-      (d: any) => equalIgnoreCase(d.dimension, id),
-    );
-
-    if (field) {
-      return field.aliasName();
-    }
-
-    field = this.measures.find(
-      (d: any) => equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id),
-    );
-
-    if (field) {
-      return field.aliasName();
-    }
-
-    return null;
-  }
-
   public groupByClause(): string {
     // QuestDB doesn't support group by index column, so map these to the alias names.
     // So, instead of:

packages/cubejs-schema-compiler/src/adapter/AWSElasticSearchQuery.ts

@@ -1,6 +1,5 @@
-import R from 'ramda';
 import { BaseFilter } from './BaseFilter';
-import { BaseQuery } from './BaseQuery';
+import { ElasticSearchQuery } from './ElasticSearchQuery';
 
 const GRANULARITY_TO_INTERVAL = {
   day: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-dd 00:00:00.000')`,
@@ -21,83 +20,24 @@ class AWSElasticSearchQueryFilter extends BaseFilter {
   }
 }
 
-export class AWSElasticSearchQuery extends BaseQuery {
-  public newFilter(filter) {
+export class AWSElasticSearchQuery extends ElasticSearchQuery {
+  public override newFilter(filter) {
     return new AWSElasticSearchQueryFilter(this, filter);
   }
 
-  public convertTz(field) {
-    return `${field}`; // TODO
-  }
-
-  public timeStampCast(value) {
-    return `${value}`;
-  }
-
-  public dateTimeCast(value) {
-    return `${value}`; // TODO
-  }
-
-  public subtractInterval(date, interval) {
+  public override subtractInterval(date, interval) {
     return `DATE_SUB(${date}, INTERVAL ${interval})`;
   }
 
-  public addInterval(date, interval) {
+  public override addInterval(date, interval) {
     return `DATE_ADD(${date}, INTERVAL ${interval})`;
   }
 
-  public timeGroupedColumn(granularity, dimension) {
+  public override timeGroupedColumn(granularity, dimension) {
     return GRANULARITY_TO_INTERVAL[granularity](dimension);
   }
 
-  public groupByClause() {
-    if (this.ungrouped) {
-      return '';
-    }
-    const dimensionsForSelect = this.dimensionsForSelect();
-    const dimensionColumns = R.flatten(dimensionsForSelect.map(s => s.selectColumns() && s.dimensionSql()))
-      .filter(s => !!s);
-    return dimensionColumns.length ? ` GROUP BY ${dimensionColumns.join(', ')}` : '';
-  }
-
-  public orderHashToString(hash) {
-    if (!hash || !hash.id) {
-      return null;
-    }
-
-    const fieldAlias = this.getFieldAlias(hash.id);
-
-    if (fieldAlias === null) {
-      return null;
-    }
-
-    const direction = hash.desc ? 'DESC' : 'ASC';
-    return `${fieldAlias} ${direction}`;
-  }
-
-  public getFieldAlias(id) {
-    const equalIgnoreCase = (a, b) => (
-      typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
-    );
-
-    let field;
-
-    field = this.dimensionsForSelect().find(d => equalIgnoreCase(d.dimension, id));
-
-    if (field) {
-      return field.dimensionSql();
-    }
-
-    field = this.measures.find(d => equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id));
-
-    if (field) {
-      return field.aliasName(); // TODO isn't supported
-    }
-
-    return null;
-  }
-
-  public escapeColumnName(name) {
-    return `${name}`; // TODO
+  public override unixTimestampSql() {
+    return 'EXTRACT(EPOCH FROM NOW())';
   }
 }