Skip to content

Commit 9b20ff4

Browse files
paveltiunovpaveltiunov
authored
fix(mssql-driver): Pre-aggregations builds hang up if over 10K of rows (#6661)
* fix(mssql-driver): Pre-aggregations builds hang up if over 10K of rows * Ignore BigECommerce in other tests for now * Add missing tables * Update snapshots * Update snapshots * Add index * Add wait strategy to address flaky behavior
1 parent 4951b69 commit 9b20ff4

24 files changed

+1298
-339
lines changed

‎packages/cubejs-mssql-driver/driver/MSSqlDriver.js‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ class MSSqlDriver extends BaseDriver {
5858
user: getEnv('dbUser', { dataSource }),
5959
password: getEnv('dbPass', { dataSource }),
6060
domain: getEnv('dbDomain', { dataSource }),
61-
requestTimeout: 10 * 60 * 1000, // 10 minutes
61+
requestTimeout: getEnv('dbQueryTimeout') * 1000,
6262
options: {
6363
encrypt: getEnv('dbSsl', { dataSource }),
6464
useUTC: false
@@ -168,7 +168,7 @@ class MSSqlDriver extends BaseDriver {
168168
case sql.Money:
169169
case sql.SmallMoney:
170170
case sql.Numeric:
171-
type = 'float';
171+
type = 'decimal';
172172
break;
173173
// double
174174
case sql.Float:
@@ -186,16 +186,16 @@ class MSSqlDriver extends BaseDriver {
186186
break;
187187
// date and time
188188
case sql.Time:
189-
type = 'date';
189+
type = 'time';
190190
break;
191191
case sql.Date:
192-
type = 'date';
192+
type = 'timestamp';
193193
break;
194194
case sql.DateTime:
195195
case sql.DateTime2:
196196
case sql.SmallDateTime:
197197
case sql.DateTimeOffset:
198-
type = 'date';
198+
type = 'timestamp';
199199
break;
200200
// others
201201
case sql.UniqueIdentifier:
@@ -213,7 +213,7 @@ class MSSqlDriver extends BaseDriver {
213213
type = 'string';
214214
break;
215215
}
216-
return { name: fields[field].name, type };
216+
return { name: fields[field].name, type: this.toGenericType(type) };
217217
});
218218
}
219219

‎packages/cubejs-mssql-driver/driver/QueryStream.js‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ const { getEnv } = require('@cubejs-backend/shared');
66
*/
77
class QueryStream extends Readable {
88
request = null;
9-
recordset = [];
10-
done = false;
9+
toRead = 0;
1110

1211
/**
1312
* @constructor
@@ -19,41 +18,43 @@ class QueryStream extends Readable {
1918
highWaterMark || getEnv('dbQueryStreamHighWaterMark'),
2019
});
2120
this.request = request;
22-
this.request.on('row', (row) => {
23-
this.recordset?.push(row);
24-
if (this.recordset?.length === getEnv('dbQueryStreamHighWaterMark')) {
21+
this.request.on('row', row => {
22+
this.transformRow(row);
23+
const canAdd = this.push(row);
24+
if (this.toRead-- <= 0 || !canAdd) {
2525
this.request.pause();
2626
}
27-
});
27+
})
28+
this.request.on('done', () => {
29+
this.push(null);
30+
})
2831
this.request.on('error', (err) => {
2932
this.destroy(err);
3033
});
31-
this.request.on('done', () => {
32-
this.done = true;
33-
});
3434
}
3535

3636
/**
3737
* @override
3838
*/
39-
_read(highWaterMark) {
40-
const chunk = this.recordset?.splice(0, highWaterMark);
41-
chunk?.forEach((row) => {
42-
this.push(row);
43-
});
44-
if (this.recordset?.length === 0 && this.done) {
45-
this.push(null);
46-
} else {
47-
this.request.resume();
39+
_read(toRead) {
40+
this.toRead += toRead;
41+
this.request.resume();
42+
}
43+
44+
transformRow(row) {
45+
for (const key in row) {
46+
if (row.hasOwnProperty(key) && row[key] && row[key] instanceof Date) {
47+
row[key] = row[key].toJSON();
48+
}
4849
}
4950
}
5051

5152
/**
5253
* @override
5354
*/
5455
_destroy(error, callback) {
56+
this.request.cancel();
5557
this.request = null;
56-
this.recordset = null;
5758
callback(error);
5859
}
5960
}

‎packages/cubejs-testing-drivers/fixtures/_schemas.json‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,104 @@
148148
"shown": false
149149
}
150150
]
151+
},
152+
{
153+
"name": "BigECommerce",
154+
"extends": "Products",
155+
"joins": [
156+
{
157+
"name": "Customers",
158+
"relationship": "belongs_to",
159+
"sql": "{CUBE}.customer_id = {Customers}.customer_id"
160+
}
161+
],
162+
"dimensions": [
163+
{
164+
"name": "id",
165+
"sql": "id",
166+
"type": "number",
167+
"primary_key": true,
168+
"shown": true
169+
},
170+
{
171+
"name": "orderId",
172+
"sql": "order_id",
173+
"type": "string"
174+
},
175+
{
176+
"name": "orderDate",
177+
"sql": "order_date",
178+
"type": "time"
179+
},
180+
{
181+
"name": "customerId",
182+
"sql": "customer_id",
183+
"type": "string"
184+
},
185+
{
186+
"name": "customerName",
187+
"sql": "{Customers.customerName}",
188+
"type":"string"
189+
},
190+
{
191+
"name": "city",
192+
"sql": "city",
193+
"type": "string"
194+
},
195+
{
196+
"name": "sales",
197+
"sql": "sales",
198+
"type":"number"
199+
},
200+
{
201+
"name": "quantity",
202+
"sql": "quantity",
203+
"type": "number"
204+
},
205+
{
206+
"name": "discount",
207+
"sql": "discount",
208+
"type": "number"
209+
},
210+
{
211+
"name": "profit",
212+
"sql": "profit",
213+
"type": "number"
214+
}
215+
],
216+
"measures": [
217+
{
218+
"name": "count",
219+
"type": "count",
220+
"sql": "customer_id"
221+
},
222+
{
223+
"name": "totalQuantity",
224+
"sql": "quantity",
225+
"type": "sum"
226+
},
227+
{
228+
"name": "avgDiscount",
229+
"sql": "discount",
230+
"type": "avg"
231+
},
232+
{
233+
"name": "totalSales",
234+
"sql": "sales",
235+
"type": "sum"
236+
},
237+
{
238+
"name": "totalProfit",
239+
"sql": "profit",
240+
"type": "sum"
241+
},
242+
{
243+
"name": "hiddenSum",
244+
"sql": "profit",
245+
"type": "sum",
246+
"shown": false
247+
}
248+
]
151249
}
152250
]
153251
}

‎packages/cubejs-testing-drivers/fixtures/athena.json‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@
2828
"tables": {
2929
"products": "products",
3030
"customers": "customers",
31-
"ecommerce": "ecommerce"
31+
"ecommerce": "ecommerce",
32+
"bigecommerce": "bigecommerce"
3233
},
3334
"preAggregations": {
3435
"Products": [],
@@ -62,6 +63,21 @@
6263
"CUBE.totalProfit"
6364
]
6465
}
66+
],
67+
"BigECommerce": [
68+
{
69+
"name": "TA",
70+
"time_dimension": "CUBE.orderDate",
71+
"granularity": "month",
72+
"partition_granularity": "year",
73+
"dimensions": ["CUBE.productName", "CUBE.id"],
74+
"measures": [
75+
"CUBE.totalQuantity",
76+
"CUBE.avgDiscount",
77+
"CUBE.totalSales",
78+
"CUBE.totalProfit"
79+
]
80+
}
6581
]
6682
},
6783
"skip": [
@@ -81,6 +97,7 @@
8197
"querying Products: dimensions -- doesn't work wo ordering",
8298
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
8399
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
84-
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test"
100+
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
101+
"querying BigECommerce: partitioned pre-agg"
85102
]
86103
}

‎packages/cubejs-testing-drivers/fixtures/bigquery.json‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@
2929
"tables": {
3030
"products": "test.products",
3131
"customers": "test.customers",
32-
"ecommerce": "test.ecommerce"
32+
"ecommerce": "test.ecommerce",
33+
"bigecommerce": "test.bigecommerce"
3334
},
3435
"preAggregations": {
3536
"Products": [],
@@ -63,6 +64,21 @@
6364
"CUBE.totalProfit"
6465
]
6566
}
67+
],
68+
"BigECommerce": [
69+
{
70+
"name": "TA",
71+
"time_dimension": "CUBE.orderDate",
72+
"granularity": "month",
73+
"partition_granularity": "year",
74+
"dimensions": ["CUBE.productName", "CUBE.id"],
75+
"measures": [
76+
"CUBE.totalQuantity",
77+
"CUBE.avgDiscount",
78+
"CUBE.totalSales",
79+
"CUBE.totalProfit"
80+
]
81+
}
6682
]
6783
},
6884
"skip": [
@@ -97,6 +113,7 @@
97113
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
98114
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
99115
"querying ECommerce: partitioned pre-agg",
100-
"querying ECommerce: partitioned pre-agg higher granularity"
116+
"querying ECommerce: partitioned pre-agg higher granularity",
117+
"querying BigECommerce: partitioned pre-agg"
101118
]
102119
}

‎packages/cubejs-testing-drivers/fixtures/clickhouse.json‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@
3737
"tables": {
3838
"products": "products",
3939
"customers": "customers",
40-
"ecommerce": "ecommerce"
40+
"ecommerce": "ecommerce",
41+
"bigecommerce": "bigecommerce"
4142
},
4243
"preAggregations": {
4344
"Products": [],
@@ -89,6 +90,26 @@
8990
}
9091
}
9192
}
93+
],
94+
"BigECommerce": [
95+
{
96+
"name": "TA",
97+
"time_dimension": "CUBE.orderDate",
98+
"granularity": "month",
99+
"partition_granularity": "year",
100+
"dimensions": ["CUBE.productName", "CUBE.id"],
101+
"measures": [
102+
"CUBE.totalQuantity",
103+
"CUBE.avgDiscount",
104+
"CUBE.totalSales",
105+
"CUBE.totalProfit"
106+
],
107+
"indexes": {
108+
"category_index": {
109+
"columns": ["CUBE.productName"]
110+
}
111+
}
112+
}
92113
]
93114
},
94115
"skip": [
@@ -99,6 +120,7 @@
99120
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
100121
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
101122
"querying ECommerce: partitioned pre-agg",
102-
"querying ECommerce: partitioned pre-agg higher granularity"
123+
"querying ECommerce: partitioned pre-agg higher granularity",
124+
"querying BigECommerce: partitioned pre-agg"
103125
]
104126
}

‎packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
"tables": {
3232
"products": "products",
3333
"customers": "customers",
34-
"ecommerce": "ecommerce"
34+
"ecommerce": "ecommerce",
35+
"bigecommerce": "bigecommerce"
3536
},
3637
"preAggregations": {
3738
"Products": [],
@@ -72,6 +73,7 @@
7273
"querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
7374
"querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
7475
"querying ECommerce: partitioned pre-agg",
75-
"querying ECommerce: partitioned pre-agg higher granularity"
76+
"querying ECommerce: partitioned pre-agg higher granularity",
77+
"querying BigECommerce: partitioned pre-agg"
7678
]
7779
}

0 commit comments

Comments
 (0)