Skip to content

Commit b90d5af

Browse files
mcheshkovmarianore-muttdata
mcheshkov
authored and
marianore-muttdata
committed
feat: Implement disable_post_processing in /v1/sql (cube-js#9331)
1 parent fed46cc commit b90d5af

File tree

14 files changed

+259
-274
lines changed

14 files changed

+259
-274
lines changed

‎docs/pages/product/apis-integrations/rest-api/reference.mdx‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -119,12 +119,6 @@ If `disable_post_processing` is set to `true`, Cube will try to generate the SQL
119119
as if the query is run without [post-processing][ref-query-wpp], i.e., if it's run as a
120120
query with [pushdown][ref-query-wpd].
121121

122-
<WarningBox>
123-
124-
Currently, the `disable_post_processing` parameter is not yet supported.
125-
126-
</WarningBox>
127-
128122
The response will contain a JSON object with the following properties under the `sql` key:
129123

130124
| Property, type | Description |

‎packages/cubejs-api-gateway/src/gateway.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,7 @@ class ApiGateway {
330330
if (req.query.format === 'sql') {
331331
await this.sql4sql({
332332
query: req.query.query,
333+
disablePostProcessing: req.query.disable_post_processing === 'true',
333334
context: req.context,
334335
res: this.resToResultFn(res)
335336
});
@@ -349,6 +350,7 @@ class ApiGateway {
349350
if (req.body.format === 'sql') {
350351
await this.sql4sql({
351352
query: req.body.query,
353+
disablePostProcessing: req.body.disable_post_processing,
352354
context: req.context,
353355
res: this.resToResultFn(res)
354356
});
@@ -1308,13 +1310,14 @@ class ApiGateway {
13081310

13091311
protected async sql4sql({
13101312
query,
1313+
disablePostProcessing,
13111314
context,
13121315
res,
1313-
}: {query: string} & BaseRequest) {
1316+
}: {query: string, disablePostProcessing: boolean} & BaseRequest) {
13141317
try {
13151318
await this.assertApiScope('data', context.securityContext);
13161319

1317-
const result = await this.sqlServer.sql4sql(query, context.securityContext);
1320+
const result = await this.sqlServer.sql4sql(query, disablePostProcessing, context.securityContext);
13181321
res({ sql: result });
13191322
} catch (e: any) {
13201323
this.handleError({

‎packages/cubejs-api-gateway/src/sql-server.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ export class SQLServer {
6464
await execSql(this.sqlInterfaceInstance!, sqlQuery, stream, securityContext);
6565
}
6666

67-
public async sql4sql(sqlQuery: string, securityContext?: any): Promise<Sql4SqlResponse> {
68-
return sql4sql(this.sqlInterfaceInstance!, sqlQuery, securityContext);
67+
public async sql4sql(sqlQuery: string, disablePostProcessing: boolean, securityContext?: unknown): Promise<Sql4SqlResponse> {
68+
return sql4sql(this.sqlInterfaceInstance!, sqlQuery, disablePostProcessing, securityContext);
6969
}
7070

7171
protected buildCheckSqlAuth(options: SQLServerOptions): CheckSQLAuthFn {

‎packages/cubejs-backend-native/js/index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -405,10 +405,10 @@ export const execSql = async (instance: SqlInterfaceInstance, sqlQuery: string,
405405
};
406406

407407
// TODO parse result from native code
408-
export const sql4sql = async (instance: SqlInterfaceInstance, sqlQuery: string, securityContext?: any): Promise<Sql4SqlResponse> => {
408+
export const sql4sql = async (instance: SqlInterfaceInstance, sqlQuery: string, disablePostProcessing: boolean, securityContext?: unknown): Promise<Sql4SqlResponse> => {
409409
const native = loadNative();
410410

411-
return native.sql4sql(instance, sqlQuery, securityContext ? JSON.stringify(securityContext) : null);
411+
return native.sql4sql(instance, sqlQuery, disablePostProcessing, securityContext ? JSON.stringify(securityContext) : null);
412412
};
413413

414414
export const buildSqlAndParams = (cubeEvaluator: any): String => {

‎packages/cubejs-backend-native/src/sql4sql.rs‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ use std::sync::Arc;
22

33
use neon::prelude::*;
44

5-
use cubesql::compile::convert_sql_to_cube_query;
65
use cubesql::compile::datafusion::logical_plan::LogicalPlan;
6+
use cubesql::compile::datafusion::scalar::ScalarValue;
7+
use cubesql::compile::datafusion::variable::VarType;
78
use cubesql::compile::engine::df::scan::CubeScanNode;
89
use cubesql::compile::engine::df::wrapper::{CubeScanWrappedSqlNode, CubeScanWrapperNode};
9-
use cubesql::sql::Session;
10+
use cubesql::compile::{convert_sql_to_cube_query, DatabaseVariable};
11+
use cubesql::sql::{Session, CUBESQL_PENALIZE_POST_PROCESSING_VAR};
1012
use cubesql::transport::MetaContext;
1113
use cubesql::CubeError;
1214

@@ -157,8 +159,20 @@ async fn handle_sql4sql_query(
157159
services: Arc<NodeCubeServices>,
158160
native_auth_ctx: Arc<NativeAuthContext>,
159161
sql_query: &str,
162+
disable_post_processing: bool,
160163
) -> Result<Sql4SqlResponse, CubeError> {
161164
with_session(&services, native_auth_ctx.clone(), |session| async move {
165+
if disable_post_processing {
166+
let v = DatabaseVariable {
167+
name: CUBESQL_PENALIZE_POST_PROCESSING_VAR.to_string(),
168+
value: ScalarValue::Boolean(Some(true)),
169+
var_type: VarType::UserDefined,
170+
readonly: false,
171+
additional_params: None,
172+
};
173+
session.state.set_variables(vec![v]);
174+
}
175+
162176
let transport = session.server.transport.clone();
163177
// todo: can we use compiler_cache?
164178
let meta_context = transport
@@ -176,8 +190,9 @@ async fn handle_sql4sql_query(
176190
pub fn sql4sql(mut cx: FunctionContext) -> JsResult<JsValue> {
177191
let interface = cx.argument::<JsBox<crate::node_export::SQLInterface>>(0)?;
178192
let sql_query = cx.argument::<JsString>(1)?.value(&mut cx);
193+
let disable_post_processing = cx.argument::<JsBoolean>(2)?.value(&mut cx);
179194

180-
let security_context: Option<serde_json::Value> = match cx.argument::<JsValue>(2) {
195+
let security_context: Option<serde_json::Value> = match cx.argument::<JsValue>(3) {
181196
Ok(string) => match string.downcast::<JsString, _>(&mut cx) {
182197
Ok(v) => v.value(&mut cx).parse::<serde_json::Value>().ok(),
183198
Err(_) => None,
@@ -208,7 +223,13 @@ pub fn sql4sql(mut cx: FunctionContext) -> JsResult<JsValue> {
208223
// can do it relatively rare, and in a single loop for all JoinHandles
209224
// this is just a watchdog for a Very Bad case, so latency requirement can be quite relaxed
210225
runtime.spawn(async move {
211-
let result = handle_sql4sql_query(services, native_auth_ctx, &sql_query).await;
226+
let result = handle_sql4sql_query(
227+
services,
228+
native_auth_ctx,
229+
&sql_query,
230+
disable_post_processing,
231+
)
232+
.await;
212233

213234
if let Err(err) = deferred.try_settle_with(&channel, move |mut cx| {
214235
// `neon::result::ResultExt` is implemented only for Result<Handle, Handle>, even though Ok variant is not touched

‎packages/cubejs-testing/test/__snapshots__/smoke-cubesql.test.ts.snap‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,64 @@ Object {
3636
}
3737
`;
3838

39+
exports[`SQL API Cube SQL over HTTP sql4sql double aggregation post-processing with disabled post-processing 1`] = `
40+
Object {
41+
"body": Object {
42+
"sql": Object {
43+
"query_type": "pushdown",
44+
"sql": Array [
45+
"SELECT \\"t\\".\\"avg_t_total_\\" \\"avg_t_total_\\"
46+
FROM (
47+
SELECT AVG(\\"t\\".\\"total\\") \\"avg_t_total_\\"
48+
FROM (
49+
SELECT
50+
\\"orders\\".status \\"status\\", sum(\\"orders\\".amount) \\"total\\"
51+
FROM
52+
(
53+
select 1 as id, 100 as amount, 'new' status, '2024-01-01'::timestamptz created_at
54+
UNION ALL
55+
select 2 as id, 200 as amount, 'new' status, '2024-01-02'::timestamptz created_at
56+
UNION ALL
57+
select 3 as id, 300 as amount, 'processed' status, '2024-01-03'::timestamptz created_at
58+
UNION ALL
59+
select 4 as id, 500 as amount, 'processed' status, '2024-01-04'::timestamptz created_at
60+
UNION ALL
61+
select 5 as id, 600 as amount, 'shipped' status, '2024-01-05'::timestamptz created_at
62+
) AS \\"orders\\" GROUP BY 1
63+
) AS \\"t\\"
64+
) AS \\"t\\"",
65+
Array [],
66+
],
67+
"status": "ok",
68+
},
69+
},
70+
"headers": Headers {
71+
Symbol(map): Object {
72+
"access-control-allow-origin": Array [
73+
"*",
74+
],
75+
"connection": Array [
76+
"keep-alive",
77+
],
78+
"content-length": Array [
79+
"878",
80+
],
81+
"content-type": Array [
82+
"application/json; charset=utf-8",
83+
],
84+
"keep-alive": Array [
85+
"timeout=5",
86+
],
87+
"x-powered-by": Array [
88+
"Express",
89+
],
90+
},
91+
},
92+
"status": 200,
93+
"statusText": "OK",
94+
}
95+
`;
96+
3997
exports[`SQL API Cube SQL over HTTP sql4sql regular query 1`] = `
4098
Object {
4199
"body": Object {
@@ -244,6 +302,42 @@ Object {
244302
}
245303
`;
246304

305+
exports[`SQL API Cube SQL over HTTP sql4sql strictly post-processing with disabled post-processing 1`] = `
306+
Object {
307+
"body": Object {
308+
"sql": Object {
309+
"error": "Provided query can not be executed without post-processing.",
310+
"query_type": "post_processing",
311+
"status": "error",
312+
},
313+
},
314+
"headers": Headers {
315+
Symbol(map): Object {
316+
"access-control-allow-origin": Array [
317+
"*",
318+
],
319+
"connection": Array [
320+
"keep-alive",
321+
],
322+
"content-length": Array [
323+
"127",
324+
],
325+
"content-type": Array [
326+
"application/json; charset=utf-8",
327+
],
328+
"keep-alive": Array [
329+
"timeout=5",
330+
],
331+
"x-powered-by": Array [
332+
"Express",
333+
],
334+
},
335+
},
336+
"status": 200,
337+
"statusText": "OK",
338+
}
339+
`;
340+
247341
exports[`SQL API Cube SQL over HTTP sql4sql wrapper 1`] = `
248342
Object {
249343
"body": Object {

‎packages/cubejs-testing/test/smoke-cubesql.test.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ describe('SQL API', () => {
149149
});
150150

151151
describe('sql4sql', () => {
152-
async function generateSql(query: string) {
152+
async function generateSql(query: string, disablePostPprocessing: boolean = false) {
153153
const response = await fetch(`${birdbox.configuration.apiUrl}/sql`, {
154154
method: 'POST',
155155
headers: {
@@ -159,6 +159,7 @@ describe('SQL API', () => {
159159
body: JSON.stringify({
160160
query,
161161
format: 'sql',
162+
disable_post_processing: disablePostPprocessing,
162163
}),
163164
});
164165
const { status, statusText, headers } = response;
@@ -193,6 +194,10 @@ describe('SQL API', () => {
193194
expect(await generateSql(`SELECT version();`)).toMatchSnapshot();
194195
});
195196

197+
it('strictly post-processing with disabled post-processing', async () => {
198+
expect(await generateSql(`SELECT version();`, true)).toMatchSnapshot();
199+
});
200+
196201
it('double aggregation post-processing', async () => {
197202
expect(await generateSql(`
198203
SELECT AVG(total)
@@ -206,6 +211,19 @@ describe('SQL API', () => {
206211
`)).toMatchSnapshot();
207212
});
208213

214+
it('double aggregation post-processing with disabled post-processing', async () => {
215+
expect(await generateSql(`
216+
SELECT AVG(total)
217+
FROM (
218+
SELECT
219+
status,
220+
SUM(totalAmount) AS total
221+
FROM Orders
222+
GROUP BY 1
223+
) t
224+
`, true)).toMatchSnapshot();
225+
});
226+
209227
it('wrapper', async () => {
210228
expect(await generateSql(`
211229
SELECT

0 commit comments

Comments
 (0)