diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts index 2332bac65f253..52a24f6df862b 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts @@ -1,6 +1,6 @@ import type { BaseQuery } from './BaseQuery'; import type { DimensionDefinition, SegmentDefinition } from '../compiler/CubeEvaluator'; -import { CubeSymbols } from "../compiler/CubeSymbols"; +import { CubeSymbols } from '../compiler/CubeSymbols'; export class BaseDimension { public readonly expression: any; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts index eb39bd3ffc52b..81d7a8c436814 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts @@ -1,7 +1,7 @@ import { UserError } from '../compiler/UserError'; import type { BaseQuery } from './BaseQuery'; import { MeasureDefinition } from '../compiler/CubeEvaluator'; -import { CubeSymbols } from "../compiler/CubeSymbols"; +import { CubeSymbols } from '../compiler/CubeSymbols'; export class BaseMeasure { public readonly expression: any; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index b87720927017a..576a2eadfe9cc 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -24,7 +24,7 @@ import { timeSeriesFromCustomInterval } from '@cubejs-backend/shared'; -import { CubeSymbols } from "../compiler/CubeSymbols"; +import { CubeSymbols } from '../compiler/CubeSymbols'; import { UserError } from '../compiler/UserError'; import { SqlParser } from '../parser/SqlParser'; import { BaseDimension } from './BaseDimension'; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts index 7a2ff39747676..2c15c80569d1d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts @@ -1,5 +1,5 @@ import type { BaseQuery } from './BaseQuery'; -import { CubeSymbols } from "../compiler/CubeSymbols"; +import { CubeSymbols } from '../compiler/CubeSymbols'; export class BaseSegment { public readonly expression: any; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts index e84a81c47d247..827396f4197c0 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseTimeDimension.ts @@ -249,17 +249,15 @@ export class BaseTimeDimension extends BaseFilter { return this.dateRangeGranularity(); } - if (!this.dateRange) { - return this.granularityObj.minGranularity(); - } - // If we have granularity and date range, we need to check // that the interval and the granularity offset are stacked/fits with date range - if (this.granularityObj.isPredefined() || - !this.granularityObj.isAlignedWithDateRange([this.dateFromFormatted(), this.dateToFormatted()])) { + if (this.dateRange && (this.granularityObj.isPredefined() || + !this.granularityObj.isAlignedWithDateRange([this.dateFromFormatted(), this.dateToFormatted()]))) { return this.query.minGranularity(this.granularityObj.minGranularity(), this.dateRangeGranularity()); } + // We return the granularity as-is, including custom ones, + // because baseQuery.granularityHierarchies correctly expands all custom granularities into hierarchies. return this.granularityObj.granularity; } ); diff --git a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts index 856382e30bf23..bf3260f040cb8 100644 --- a/packages/cubejs-schema-compiler/src/adapter/Granularity.ts +++ b/packages/cubejs-schema-compiler/src/adapter/Granularity.ts @@ -31,7 +31,7 @@ export class Granularity { ) { this.granularity = timeDimension.granularity; this.predefinedGranularity = isPredefinedGranularity(this.granularity); - this.queryTimezone = query.timezone; + this.queryTimezone = query.timezone || 'UTC'; this.origin = moment.tz(query.timezone).startOf('year'); // Defaults to current year start if (this.predefinedGranularity) { diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index 60f0a459e8f25..71a829546ed2b 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -1,7 +1,7 @@ import R from 'ramda'; import { FROM_PARTITION_RANGE, getEnv, TO_PARTITION_RANGE } from '@cubejs-backend/shared'; -import { CubeSymbols } from "../compiler/CubeSymbols"; +import { CubeSymbols } from '../compiler/CubeSymbols'; import { UserError } from '../compiler/UserError'; export class PreAggregations { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index a19cd02de00cb..f734e969fac49 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -32,6 +32,10 @@ describe('PreAggregations', () => { type: 'count' }, + countAnother: { + type: 'count' + }, + checkinsTotal: { sql: \`\${checkinsCount}\`, type: 'sum' @@ -100,6 +104,10 @@ describe('PreAggregations', () => { hourTenMinOffset: { interval: '1 hour', offset: '10 minutes' + }, + halfYear: { + interval: '6 months', + origin: '2017-01-01' } } }, @@ -237,6 +245,12 @@ describe('PreAggregations', () => { granularity: 'hourTenMinOffset', allowNonStrictDateRangeMatch: false }, + countAnotherCountCustomGranularity: { + measures: [countAnother], + timeDimension: createdAt, + granularity: 'halfYear', + allowNonStrictDateRangeMatch: false + }, sourceAndIdRollup: { measures: [count], dimensions: [sourceAndId, source], @@ -496,7 +510,9 @@ describe('PreAggregations', () => { }); `); - it('simple pre-aggregation', () => compiler.compile().then(() => { + it('simple pre-aggregation', async () => { + await compiler.compile(); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.count' @@ -535,9 +551,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('simple pre-aggregation (allowNonStrictDateRangeMatch: true)', async () => { + await compiler.compile(); - it('simple pre-aggregation (allowNonStrictDateRangeMatch: true)', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.count' @@ -578,9 +596,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('simple pre-aggregation with custom granularity (exact match)', async () => { + await compiler.compile(); - it('simple pre-aggregation with custom granularity (exact match)', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.count' @@ -621,9 +641,68 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('simple pre-aggregation with custom granularity (exact match) 2', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'visitors.countAnother' + ], + timeDimensions: [{ + dimension: 'visitors.createdAt', + dateRange: ['2017-01-01 00:00:00.000', '2017-12-31 23:59:59.999'], + granularity: 'halfYear', + }], + timezone: 'UTC', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(queryAndParams[0]).toMatch(/visitors_count_another_count_custom_granularity/); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual([ + { + visitors__count_another: '5', + visitors__created_at_halfYear: '2017-01-01T00:00:00.000Z', + }, + ]); + }); + }); + + it('pre-aggregation with custom granularity should match its own references', async () => { + await compiler.compile(); + + const preAggregationId = 'visitors.countAnotherCountCustomGranularity'; + const preAggregations = cubeEvaluator.preAggregations({ preAggregationIds: [preAggregationId] }); + + const preAggregation = preAggregations[0]; + if (preAggregation === undefined) { + throw expect(preAggregation).toBeDefined(); + } + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + ...preAggregation.references, + preAggregationId: preAggregation.id, + timezone: 'UTC', + }); + + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + const preAggregationFromQuery = preAggregationsDescription.find(p => p.preAggregationId === preAggregation.id); + if (preAggregationFromQuery === undefined) { + throw expect(preAggregationFromQuery).toBeDefined(); + } + + expect(preAggregationFromQuery.preAggregationId).toBe(preAggregationId); + }); + + it('leaf measure pre-aggregation', async () => { + await compiler.compile(); - it('leaf measure pre-aggregation', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.ratio' @@ -668,9 +747,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('leaf measure view pre-aggregation', async () => { + await compiler.compile(); - it('leaf measure view pre-aggregation', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors_view.ratio' @@ -715,9 +796,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('non-additive measure view pre-aggregation', async () => { + await compiler.compile(); - it('non-additive measure view pre-aggregation', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors_view.uniqueSourceCount' @@ -762,9 +845,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('non-additive single value view filter', async () => { + await compiler.compile(); - it('non-additive single value view filter', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors_view.uniqueSourceCount' @@ -802,9 +887,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('non-additive single value view filtered measure', async () => { + await compiler.compile(); - it('non-additive single value view filtered measure', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors_view.googleUniqueSourceCount' @@ -842,9 +929,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('multiplied measure no match', async () => { + await compiler.compile(); - it('multiplied measure no match', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.count' @@ -878,9 +967,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('multiplied measure match', async () => { + await compiler.compile(); - it('multiplied measure match', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.count' @@ -937,9 +1028,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('non-leaf additive measure', async () => { + await compiler.compile(); - it('non-leaf additive measure', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors_view.count' @@ -980,9 +1073,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('non-leaf additive measure with time dimension', async () => { + await compiler.compile(); - it('non-leaf additive measure with time dimension', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors_view.count' @@ -1034,9 +1129,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('inherited original sql', async () => { + await compiler.compile(); - it('inherited original sql', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'GoogleVisitors.count' @@ -1068,9 +1165,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('immutable partition default refreshKey', async () => { + await compiler.compile(); - it('immutable partition default refreshKey', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'GoogleVisitors.checkinsTotal' @@ -1108,9 +1207,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('immutable every hour', async () => { + await compiler.compile(); - it('immutable every hour', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'EveryHourVisitors.checkinsTotal' @@ -1151,9 +1252,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('reference original sql', async () => { + await compiler.compile(); - it('reference original sql', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'ReferenceOriginalSql.count' @@ -1192,9 +1295,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('partitioned scheduled refresh', async () => { + await compiler.compile(); - it('partitioned scheduled refresh', () => compiler.compile().then(async () => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitor_checkins.count' @@ -1226,9 +1331,11 @@ describe('PreAggregations', () => { expect(res).toEqual( [{ max: '2017-01-06T00:00:00.000Z' }] ); - })); + }); + + it('empty scheduled refresh', async () => { + await compiler.compile(); - it('empty scheduled refresh', () => compiler.compile().then(async () => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitor_checkins.count' @@ -1258,9 +1365,11 @@ describe('PreAggregations', () => { expect(res).toEqual( [{ max: null }] ); - })); + }); + + it('mutable partition default refreshKey', async () => { + await compiler.compile(); - it('mutable partition default refreshKey', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1313,9 +1422,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('hll bigquery rollup', async () => { + await compiler.compile(); - it('hll bigquery rollup', () => compiler.compile().then(() => { const query = new BigqueryQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.countDistinctApprox' @@ -1339,9 +1450,11 @@ describe('PreAggregations', () => { expect(queryAndParams[0]).toMatch(/HLL_COUNT\.MERGE/); expect(preAggregationsDescription.loadSql[0]).toMatch(/HLL_COUNT\.INIT/); - })); + }); + + it('sub query', async () => { + await compiler.compile(); - it('sub query', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.count' @@ -1373,9 +1486,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('multi-stage', async () => { + await compiler.compile(); - it('multi-stage', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1418,9 +1533,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('incremental renewal threshold', async () => { + await compiler.compile(); - it('incremental renewal threshold', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1467,9 +1584,11 @@ describe('PreAggregations', () => { }] ); }); - })); + }); + + it('partitioned', async () => { + await compiler.compile(); - it('partitioned', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1520,9 +1639,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('partitioned inDateRange', async () => { + await compiler.compile(); - it('partitioned inDateRange', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1566,9 +1687,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('partitioned hourly', async () => { + await compiler.compile(); - it('partitioned hourly', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1616,9 +1739,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('partitioned rolling', async () => { + await compiler.compile(); - it('partitioned rolling', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsRollingTotal', @@ -1691,9 +1816,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('partitioned rolling 2 day', async () => { + await compiler.compile(); - it('partitioned rolling 2 day', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsRolling2day', @@ -1751,9 +1878,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('not aligned time dimension', async () => { + await compiler.compile(); - it('not aligned time dimension', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1800,9 +1929,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('segment', async () => { + await compiler.compile(); - it('segment', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1843,9 +1974,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('rollup join', async () => { + await compiler.compile(); - it('rollup join', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitor_checkins.count', @@ -1878,7 +2011,7 @@ describe('PreAggregations', () => { ], ); }); - })); + }); it('rollup join existing joins', async () => { await compiler.compile(); @@ -1920,7 +2053,9 @@ describe('PreAggregations', () => { }); }); - it('rollup join partitioned', () => compiler.compile().then(() => { + it('rollup join partitioned', async () => { + await compiler.compile(); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitor_checkins.count', @@ -1961,9 +2096,11 @@ describe('PreAggregations', () => { ], ); }); - })); + }); + + it('partitioned without time', async () => { + await compiler.compile(); - it('partitioned without time', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors.checkinsTotal' @@ -1995,9 +2132,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('partitioned huge span', async () => { + await compiler.compile(); - it('partitioned huge span', () => compiler.compile().then(() => { let queryAndParams; let preAggregationsDescription; let query; @@ -2059,9 +2198,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('simple view', async () => { + await compiler.compile(); - it('simple view', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors_view.checkinsTotal' @@ -2114,9 +2255,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('simple view non matching time-dimension granularity', async () => { + await compiler.compile(); - it('simple view non matching time-dimension granularity', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'visitors_view.checkinsTotal' @@ -2164,9 +2307,11 @@ describe('PreAggregations', () => { ] ); }); - })); + }); + + it('lambda cross data source refresh key and ungrouped', async () => { + await compiler.compile(); - it('lambda cross data source refresh key and ungrouped', () => compiler.compile().then(() => { const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { measures: [ 'LambdaVisitors.count' @@ -2206,5 +2351,5 @@ describe('PreAggregations', () => { expect(partitionInvalidateKeyQueries).toStrictEqual([]); expect(loadSql[0]).not.toMatch(/GROUP BY/); expect(loadSql[0]).toMatch(/THEN 1 END `real_time_lambda_visitors__count`/); - })); + }); }); diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index d2c23af1dfb97..a72aa7df8c943 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -1,7 +1,7 @@ import { PostgresQuery } from '../../src/adapter/PostgresQuery'; import { prepareCube } from './PrepareCompiler'; import { PreAggregations } from '../../src/adapter/PreAggregations'; -import { PreAggregationReferences } from "../../src/compiler/CubeEvaluator"; +import { PreAggregationReferences } from '../../src/compiler/CubeEvaluator'; describe('Pre Aggregation by filter match tests', () => { function getCube(cube) {