feat(cubesql): Add ability to filter dates inclusive of date being passed in when using <= or >= (#7041) Thanks @darapuk !

darapuk · web-flow · commit 6b9ae703b113 · 2023-09-19T22:16:18.000-07:00
* (Fix) Add ability to filter dates inclusive of date being passed in when using &lt;= or &gt;=

* fix test + fmt

* missed fmt

* Fix date_trunc

* revert/patch tests

* Fix tests

* consolidate code

* fix test

* fix tests

* Make sure valid filters are passed in + sort dates correctly
diff --git a/examples/apollo-federation-with-cube/apollo-server-app/graphql/cube.graphql b/examples/apollo-federation-with-cube/apollo-server-app/graphql/cube.graphql
@@ -33,7 +33,9 @@ input DateTimeFilter {
   inDateRange: [String]
   notInDateRange: [String]
   beforeDate: String
+  beforeOrOnDate: String
   afterDate: String
+  afterOrOnDate: String
   set: Boolean
 }
 
diff --git a/examples/apollo-federation-with-cube/apollo-server-app/graphql/supergraph.graphql b/examples/apollo-federation-with-cube/apollo-server-app/graphql/supergraph.graphql
@@ -32,7 +32,9 @@ input DateTimeFilter
   inDateRange: [String]
   notInDateRange: [String]
   beforeDate: String
+  beforeOrOnDate: String
   afterDate: String
+  afterOrOnDate: String
   set: Boolean
 }
 
diff --git a/packages/cubejs-api-gateway/src/graphql.ts b/packages/cubejs-api-gateway/src/graphql.ts
@@ -77,7 +77,9 @@ const DateTimeFilter = inputObjectType({
     t.list.string('inDateRange');
     t.list.string('notInDateRange');
     t.string('beforeDate');
+    t.string('beforeOrOnDate');
     t.string('afterDate');
+    t.string('afterOrOnDate');
     t.boolean('set');
   }
 });
diff --git a/packages/cubejs-api-gateway/src/query.js b/packages/cubejs-api-gateway/src/query.js
@@ -69,7 +69,9 @@ const operators = [
   'notInDateRange',
   'onTheDate',
   'beforeDate',
+  'beforeOrOnDate',
   'afterDate',
+  'afterOrOnDate',
   'measureFilter',
 ];
 
diff --git a/packages/cubejs-api-gateway/src/types/strings.ts b/packages/cubejs-api-gateway/src/types/strings.ts
@@ -75,7 +75,9 @@ type FilterOperator =
  'notInDateRange' |
  'onTheDate' |
  'beforeDate' |
+ 'beforeOrOnDate' |
  'afterDate' |
+ 'afterOrOnDate' |
  'measureFilter';
 
 /**
diff --git a/packages/cubejs-client-core/index.d.ts b/packages/cubejs-client-core/index.d.ts
@@ -774,7 +774,9 @@ declare module '@cubejs-client/core' {
     | 'inDateRange'
     | 'notInDateRange'
     | 'beforeDate'
-    | 'afterDate';
+    | 'beforeOrOnDate'
+    | 'afterDate'
+    | 'afterOrOnDate';
 
   export type TimeDimensionGranularity = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year';
 
diff --git a/packages/cubejs-client-core/src/Meta.js b/packages/cubejs-client-core/src/Meta.js
@@ -35,7 +35,9 @@ const operators = {
     { name: 'inDateRange', title: 'in date range' },
     { name: 'notInDateRange', title: 'not in date range' },
     { name: 'afterDate', title: 'after date' },
+    { name: 'afterOrOnDate', title: 'after or on date' },
     { name: 'beforeDate', title: 'before date' },
+    { name: 'beforeOrOnDate', title: 'before or on date' },
   ],
 };
 
diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.js b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.js
@@ -7,7 +7,7 @@ import { BaseDimension } from './BaseDimension';
 
 const moment = momentRange.extendMoment(require('moment-timezone'));
 
-const DATE_OPERATORS = ['inDateRange', 'notInDateRange', 'onTheDate', 'beforeDate', 'afterDate'];
+const DATE_OPERATORS = ['inDateRange', 'notInDateRange', 'onTheDate', 'beforeDate', 'beforeOrOnDate', 'afterDate', 'afterOrOnDate'];
 const dateTimeLocalMsRegex = /^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\d$/;
 const dateRegex = /^\d\d\d\d-\d\d-\d\d$/;
 
@@ -332,11 +332,21 @@ export class BaseFilter extends BaseDimension {
     return this.query.beforeDateFilter(column, before);
   }
 
+  beforeOrOnDateWhere(column) {
+    const [before] = this.allocateTimestampParams();
+    return this.query.beforeOrOnDateFilter(column, before);
+  }
+
   afterDateWhere(column) {
     const [after] = this.allocateTimestampParams();
     return this.query.afterDateFilter(column, after);
   }
 
+  afterOrOnDateWhere(column) {
+    const [after] = this.allocateTimestampParams();
+    return this.query.afterOrOnDateFilter(column, after);
+  }
+
   formatFromDate(date) {
     if (date && date.match(dateTimeLocalMsRegex)) {
       return date;
diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js
@@ -1074,10 +1074,18 @@ class BaseQuery {
     return `${dimensionSql} < ${timeStampParam}`;
   }
 
+  beforeOrOnDateFilter(dimensionSql, timeStampParam) {
+    return `${dimensionSql} <= ${timeStampParam}`;
+  }
+
   afterDateFilter(dimensionSql, timeStampParam) {
     return `${dimensionSql} > ${timeStampParam}`;
   }
 
+  afterOrOnDateFilter(dimensionSql, timeStampParam) {
+    return `${dimensionSql} >= ${timeStampParam}`;
+  }
+
   timeStampCast(value) {
     return `${value}::timestamptz`;
   }
diff --git a/rust/cubesql/cubesql/src/compile/legacy_compiler.rs b/rust/cubesql/cubesql/src/compile/legacy_compiler.rs
@@ -557,7 +557,9 @@ fn compiled_binary_op_expr(
                 ast::BinaryOperator::Eq => (filter_expr, "equals".to_string()),
                 ast::BinaryOperator::NotEq => (filter_expr, "notEquals".to_string()),
                 ast::BinaryOperator::GtEq => match filter_expr {
-                    CompiledExpression::DateLiteral(_) => (filter_expr, "afterDate".to_string()),
+                    CompiledExpression::DateLiteral(_) => {
+                        (filter_expr, "afterOrOnDate".to_string())
+                    }
                     _ => (filter_expr, "gte".to_string()),
                 },
                 ast::BinaryOperator::Gt => match filter_expr {
@@ -575,7 +577,9 @@ fn compiled_binary_op_expr(
                     _ => (filter_expr, "lt".to_string()),
                 },
                 ast::BinaryOperator::LtEq => match filter_expr {
-                    CompiledExpression::DateLiteral(_) => (filter_expr, "beforeDate".to_string()),
+                    CompiledExpression::DateLiteral(_) => {
+                        (filter_expr, "beforeOrOnDate".to_string())
+                    }
                     _ => (filter_expr, "lte".to_string()),
                 },
                 _ => {
@@ -742,7 +746,11 @@ fn binary_op_create_node_and(
                         && ((l_op.eq(&"beforeDate".to_string())
                             && r_op.eq(&"afterDate".to_string()))
                             || (l_op.eq(&"afterDate".to_string())
-                                && r_op.eq(&"beforeDate".to_string())))
+                                && r_op.eq(&"beforeDate".to_string()))
+                            || (l_op.eq(&"afterOrOnDate".to_string())
+                                && r_op.eq(&"beforeOrOnDate".to_string()))
+                            || (l_op.eq(&"beforeOrOnDate".to_string())
+                                && r_op.eq(&"afterOrOnDate".to_string())))
                     {
                         return Ok(CompiledFilterTree::Filter(CompiledFilter::Filter {
                             member: l_member.clone(),
diff --git a/rust/cubesql/cubesql/src/compile/mod.rs b/rust/cubesql/cubesql/src/compile/mod.rs
@@ -3059,7 +3059,7 @@ mod tests {
                     granularity: None,
                     date_range: Some(json!(vec![
                         "2020-12-25T22:48:48.000Z".to_string(),
-                        "2022-03-31T23:59:59.999Z".to_string()
+                        "2022-04-01T00:00:00.000Z".to_string()
                     ]))
                 }]),
                 order: None,
@@ -4281,15 +4281,15 @@ ORDER BY \"COUNT(count)\" DESC"
                 or: Some(vec![
                     json!(V1LoadRequestQueryFilterItem {
                         member: Some("KibanaSampleDataEcommerce.order_date".to_string()),
-                        operator: Some("afterDate".to_string()),
+                        operator: Some("afterOrOnDate".to_string()),
                         values: Some(vec!["2021-08-31T00:00:00.000Z".to_string()]),
                         or: None,
                         and: None,
                     }),
                     json!(V1LoadRequestQueryFilterItem {
                         member: Some("KibanaSampleDataEcommerce.order_date".to_string()),
                         operator: Some("beforeDate".to_string()),
-                        values: Some(vec!["2021-09-06T23:59:59.999Z".to_string()]),
+                        values: Some(vec!["2021-09-07T00:00:00.000Z".to_string()]),
                         or: None,
                         and: None,
                     })
@@ -11614,7 +11614,7 @@ ORDER BY \"COUNT(count)\" DESC"
                 offset: None,
                 filters: Some(vec![V1LoadRequestQueryFilterItem {
                     member: Some("KibanaSampleDataEcommerce.order_date".to_string()),
-                    operator: Some("afterDate".to_string()),
+                    operator: Some("afterOrOnDate".to_string()),
                     values: filter_vals,
                     or: None,
                     and: None,
@@ -13935,7 +13935,7 @@ ORDER BY \"COUNT(count)\" DESC"
                     granularity: Some("day".to_string()),
                     date_range: Some(json!(vec![
                         "2020-01-01T00:00:00.000Z".to_string(),
-                        format!("{}T23:59:59.999Z", end_date),
+                        format!("{}T00:00:00.000Z", end_date),
                     ]))
                 }]),
                 order: Some(vec![vec![
@@ -17333,7 +17333,7 @@ ORDER BY \"COUNT(count)\" DESC"
                 filters: Some(vec![V1LoadRequestQueryFilterItem {
                     member: Some("KibanaSampleDataEcommerce.order_date".to_string()),
                     operator: Some("beforeDate".to_string()),
-                    values: Some(vec!["2014-05-30T23:59:59.999Z".to_string()]),
+                    values: Some(vec!["2014-05-31T00:00:00.000Z".to_string()]),
                     or: None,
                     and: None
                 }]),
@@ -17909,18 +17909,48 @@ ORDER BY \"COUNT(count)\" DESC"
 
         let test_data = vec![
             // (operator, literal date, filter operator, filter value)
-            (">=", "2020-03-25", "afterDate", "2020-04-01T00:00:00.000Z"),
-            (">=", "2020-04-01", "afterDate", "2020-04-01T00:00:00.000Z"),
-            (">=", "2020-04-10", "afterDate", "2020-05-01T00:00:00.000Z"),
-            ("<=", "2020-03-25", "beforeDate", "2020-03-31T23:59:59.999Z"),
-            ("<=", "2020-04-01", "beforeDate", "2020-04-30T23:59:59.999Z"),
-            ("<=", "2020-04-10", "beforeDate", "2020-04-30T23:59:59.999Z"),
+            (
+                ">=",
+                "2020-03-25",
+                "afterOrOnDate",
+                "2020-04-01T00:00:00.000Z",
+            ),
+            (
+                ">=",
+                "2020-04-01",
+                "afterOrOnDate",
+                "2020-04-01T00:00:00.000Z",
+            ),
+            (
+                ">=",
+                "2020-04-10",
+                "afterOrOnDate",
+                "2020-05-01T00:00:00.000Z",
+            ),
+            (
+                "<=",
+                "2020-03-25",
+                "beforeOrOnDate",
+                "2020-04-01T00:00:00.000Z",
+            ),
+            (
+                "<=",
+                "2020-04-01",
+                "beforeOrOnDate",
+                "2020-05-01T00:00:00.000Z",
+            ),
+            (
+                "<=",
+                "2020-04-10",
+                "beforeOrOnDate",
+                "2020-05-01T00:00:00.000Z",
+            ),
             (">", "2020-03-25", "afterDate", "2020-04-01T00:00:00.000Z"),
             (">", "2020-04-01", "afterDate", "2020-05-01T00:00:00.000Z"),
             (">", "2020-04-10", "afterDate", "2020-05-01T00:00:00.000Z"),
-            ("<", "2020-03-25", "beforeDate", "2020-03-31T23:59:59.999Z"),
-            ("<", "2020-04-01", "beforeDate", "2020-03-31T23:59:59.999Z"),
-            ("<", "2020-04-10", "beforeDate", "2020-04-30T23:59:59.999Z"),
+            ("<", "2020-03-25", "beforeDate", "2020-04-01T00:00:00.000Z"),
+            ("<", "2020-04-01", "beforeDate", "2020-04-01T00:00:00.000Z"),
+            ("<", "2020-04-10", "beforeDate", "2020-05-01T00:00:00.000Z"),
         ];
 
         for (operator, literal_date, filter_operator, filter_value) in test_data {
diff --git a/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs b/rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs

Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,9 @@ input DateTimeFilter {`
`33`	`33`	`inDateRange: [String]`
`34`	`34`	`notInDateRange: [String]`
`35`	`35`	`beforeDate: String`
	`36`	`+ beforeOrOnDate: String`
`36`	`37`	`afterDate: String`
	`38`	`+ afterOrOnDate: String`
`37`	`39`	`set: Boolean`
`38`	`40`	`}`
`39`	`41`
Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,9 @@ input DateTimeFilter`
`32`	`32`	`inDateRange: [String]`
`33`	`33`	`notInDateRange: [String]`
`34`	`34`	`beforeDate: String`
	`35`	`+ beforeOrOnDate: String`
`35`	`36`	`afterDate: String`
	`37`	`+ afterOrOnDate: String`
`36`	`38`	`set: Boolean`
`37`	`39`	`}`
`38`	`40`
Original file line number	Diff line number	Diff line change
`@@ -1074,10 +1074,18 @@ class BaseQuery {`
`1074`	`1074`	return `${dimensionSql} < ${timeStampParam}`;
`1075`	`1075`	`}`
`1076`	`1076`
	`1077`	`+ beforeOrOnDateFilter(dimensionSql, timeStampParam) {`
	`1078`	+ return `${dimensionSql} <= ${timeStampParam}`;
	`1079`	`+ }`
	`1080`	`+`
`1077`	`1081`	`afterDateFilter(dimensionSql, timeStampParam) {`
`1078`	`1082`	return `${dimensionSql} > ${timeStampParam}`;
`1079`	`1083`	`}`
`1080`	`1084`
	`1085`	`+ afterOrOnDateFilter(dimensionSql, timeStampParam) {`
	`1086`	+ return `${dimensionSql} >= ${timeStampParam}`;
	`1087`	`+ }`
	`1088`	`+`
`1081`	`1089`	`timeStampCast(value) {`
`1082`	`1090`	return `${value}::timestamptz`;
`1083`	`1091`	`}`