feat: Hour partition granularity support

Author: paveltiunov

packages/cubejs-schema-compiler/adapter/PreAggregations.js

@@ -134,12 +134,17 @@ class PreAggregations {
   }
 
   preAggregationTableName(cube, preAggregationName, preAggregation, skipSchema) {
+    let partitionSuffix = '';
+    if (preAggregation.partitionTimeDimensions) {
+      const partitionTimeDimension = preAggregation.partitionTimeDimensions[0];
+      partitionSuffix = partitionTimeDimension.dateRange[0].substring(
+        0,
+        preAggregation.partitionGranularity === 'hour' ? 13 : 10
+      ).replace(/[-T:]/g, '');
+    }
     return this.query.preAggregationTableName(
-      cube, preAggregationName + (
-        preAggregation.partitionTimeDimensions ?
-          preAggregation.partitionTimeDimensions[0].dateRange[0].replace('T00:00:00.000', '').replace(/-/g, '') :
-          ''
-      ),
+      cube,
+      preAggregationName + partitionSuffix,
       skipSchema
     );
   }

packages/cubejs-schema-compiler/compiler/CubeValidator.js

@@ -74,7 +74,7 @@ const BasePreAggregation = {
   ),
   useOriginalSqlPreAggregations: Joi.boolean(),
   external: Joi.boolean(),
-  partitionGranularity: Joi.any().valid('day', 'week', 'month', 'year'),
+  partitionGranularity: Joi.any().valid('hour', 'day', 'week', 'month', 'year'),
   scheduledRefresh: Joi.boolean(),
   indexes: Joi.object().pattern(identifierRegex, Joi.alternatives().try(
     Joi.object().keys({

packages/cubejs-schema-compiler/parser/SqlParser.js

@@ -73,13 +73,9 @@ class SqlParser {
         return sql.slice(start, stop + 1);
       }
     };
-    const lexer = new GenericSqlLexer(chars);
-    const tokens = new antlr4.CommonTokenStream(lexer);
-    const parser = new GenericSqlParser(tokens);
-    parser.buildParseTrees = true;
+
     const errors = [];
     this.errors = errors;
-    parser.removeErrorListeners();
 
     class ExprErrorListener extends antlr4.error.ErrorListener {
       syntaxError(recognizer, offendingSymbol, line, column, msg, err) {
@@ -89,6 +85,13 @@ class SqlParser {
       }
     }
 
+    const lexer = new GenericSqlLexer(chars);
+    lexer.removeErrorListeners();
+    lexer.addErrorListener(new ExprErrorListener());
+    const tokens = new antlr4.CommonTokenStream(lexer);
+    const parser = new GenericSqlParser(tokens);
+    parser.buildParseTrees = true;
+    parser.removeErrorListeners();
     parser.addErrorListener(new ExprErrorListener());
 
     return parser.statement();

packages/cubejs-schema-compiler/test/PreAggregationsTest.js

@@ -1,12 +1,13 @@
 /* eslint-disable quote-props */
+/* globals it, describe, after */
 const R = require('ramda');
 
 const PostgresQuery = require('../adapter/PostgresQuery');
 const BigqueryQuery = require('../adapter/BigqueryQuery');
 const PrepareCompiler = require('./PrepareCompiler');
 require('should');
 
-const prepareCompiler = PrepareCompiler.prepareCompiler;
+const { prepareCompiler } = PrepareCompiler;
 const dbRunner = require('./DbRunner');
 
 describe('PreAggregations', function test() {
@@ -128,6 +129,14 @@ describe('PreAggregations', function test() {
           granularity: 'day',
           partitionGranularity: 'month'
         },
+        partitionedHourly: {
+          type: 'rollup',
+          measureReferences: [checkinsTotal],
+          dimensionReferences: [source],
+          timeDimensionReference: createdAt,
+          granularity: 'hour',
+          partitionGranularity: 'hour'
+        },
         ratio: {
           type: 'rollup',
           measureReferences: [checkinsTotal, uniqueSourceCount],
@@ -749,6 +758,58 @@ describe('PreAggregations', function test() {
     });
   });
 
+  it('partitioned hourly', () => {
+    return compiler.compile().then(() => {
+      const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
+        measures: [
+          'visitors.checkinsTotal'
+        ],
+        dimensions: [
+          'visitors.source'
+        ],
+        timezone: 'UTC',
+        preAggregationsSchema: '',
+        timeDimensions: [{
+          dimension: 'visitors.createdAt',
+          granularity: 'hour',
+          dateRange: ['2017-01-02', '2017-01-05']
+        }],
+        order: [{
+          id: 'visitors.createdAt'
+        }],
+      });
+
+      const queryAndParams = query.buildSqlAndParams();
+      console.log(queryAndParams);
+      const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
+      console.log(preAggregationsDescription);
+
+      const queries = tempTablePreAggregations(preAggregationsDescription);
+
+      console.log(JSON.stringify(queries.concat(queryAndParams)));
+
+      return dbRunner.testQueries(
+        queries.concat([queryAndParams]).map(q => replaceTableName(q, preAggregationsDescription, 242))
+      ).then(res => {
+        console.log(JSON.stringify(res));
+        res.should.be.deepEqual(
+          [
+            {
+              "visitors__source": "some",
+              "visitors__created_at_hour": "2017-01-03T00:00:00.000Z",
+              "visitors__checkins_total": "3"
+            },
+            {
+              "visitors__source": "some",
+              "visitors__created_at_hour": "2017-01-05T00:00:00.000Z",
+              "visitors__checkins_total": "2"
+            }
+          ]
+        );
+      });
+    });
+  });
+
   it('segment', () => {
     return compiler.compile().then(() => {
       const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {