make all native cubestore_result_transform functions async

Author: KSDaemon

packages/cubejs-api-gateway/src/gateway.ts

@@ -1588,7 +1588,7 @@ class ApiGateway {
    * result object.
    * @internal
    */
-  private getResultInternal(
+  private async getResultInternal(
     context: RequestContext,
     queryType: QueryType,
     normalizedQuery: NormalizedQuery,
@@ -1625,13 +1625,14 @@ class ApiGateway {
     // We postpone data transformation until the last minute
     // in case when all responses are native - we process them in native part
     const dataCb: TransformDataResponseCb = response.data.isNative ?
-      () => JSON.parse(
-        transformDataNative(
+      async () => {
+        const jsonData = await transformDataNative(
           JSON.stringify(transformDataParams), response.data.getNativeRef()
-        ).result
-      ) as TransformDataResponse
+        );
+        return JSON.parse(jsonData.result) as TransformDataResponse;
+      }
       :
-      () => transformData({
+      async () => transformData({
         ...transformDataParams,
         data: response.data,
       });
@@ -1842,20 +1843,20 @@ class ApiGateway {
           };
           const resultDataJson = JSON.stringify(responseDataObj);
 
-          res(getFinalCubestoreResultMulti(transformDataJson, rawDataRef, resultDataJson));
+          res(await getFinalCubestoreResultMulti(transformDataJson, rawDataRef, resultDataJson));
         } else {
           // if we have mixed query results (there are js and native)
           // we prepare results separately: on js and native sides
           // and serve final response from JS side
           res({
             queryType,
-            results: results.map(r => {
-              const data = r.dataCb();
+            results: await Promise.all(results.map(async (r) => {
+              const data = await r.dataCb();
               return {
                 ...cleanupResult(r),
                 data,
               };
-            }),
+            })),
             pivotQuery: getPivotQuery(queryType, normalizedQueries),
             slowQuery
           });
@@ -1866,9 +1867,9 @@ class ApiGateway {
         const transformDataJson = JSON.stringify(r.transformDataParams);
         const rawDataRef = r.rawData.getNativeRef();
         const resultDataJson = JSON.stringify(cleanupResult(r));
-        res(getFinalCubestoreResult(transformDataJson, rawDataRef, resultDataJson));
+        res(await getFinalCubestoreResult(transformDataJson, rawDataRef, resultDataJson));
       } else {
-        const data = results[0].dataCb();
+        const data = await results[0].dataCb();
         res({
           ...cleanupResult(results[0]),
           data,
@@ -2021,19 +2022,19 @@ class ApiGateway {
               [[], [], []]
             );
 
-            res(getFinalCubestoreResultArray(transformDataJson, rawDataRef, resultDataJson));
+            res(await getFinalCubestoreResultArray(transformDataJson, rawDataRef, resultDataJson));
           } else {
             // if we have mixed query results (there are js and native)
             // we prepare results separately: on js and native sides
             // and serve final response from JS side
             res({
-              results: results.map(r => {
-                const data = r.dataCb();
+              results: await Promise.all(results.map(async (r) => {
+                const data = await r.dataCb();
                 return {
                   ...cleanupResult(r),
                   data,
                 };
-              }),
+              })),
             });
           }
         } else {

packages/cubejs-api-gateway/src/types/responses.ts

@@ -34,4 +34,4 @@ export type TransformDataRequest = {
   resType?: ResultType
 };
 
-export type TransformDataResponseCb = () => TransformDataResponse;
+export type TransformDataResponseCb = () => Promise<TransformDataResponse>;

packages/cubejs-backend-native/Cargo.lock

@@ -19,6 +19,7 @@ crate-type = ["cdylib", "lib"]
 cubesqlplanner = { path = "../../rust/cubesqlplanner/cubesqlplanner" }
 cubeorchestrator = { path = "../../rust/cubeorchestrator" }
 cubenativeutils = { path = "../../rust/cubenativeutils" }
+anyhow = "1.0"
 async-channel = { version = "2" }
 async-trait = "0.1.36"
 convert_case = "0.6.0"

packages/cubejs-backend-native/Cargo.toml

@@ -383,25 +383,25 @@ export const getCubestoreResult = (ref: CubeStoreResultWrapper): ResultRow[] =>
   return native.getCubestoreResult(ref);
 };
 
-export const transformData = (transformDataJson: string, rows: any): TransformDataResponseNative => {
+export const transformData = (transformDataJson: string, rows: any): Promise<TransformDataResponseNative> => {
   const native = loadNative();
 
   return native.transformQueryData(transformDataJson, rows);
 };
 
-export const getFinalCubestoreResult = (transformDataJson: string, rows: any, resultData: string): ArrayBuffer => {
+export const getFinalCubestoreResult = (transformDataJson: string, rows: any, resultData: string): Promise<ArrayBuffer> => {
   const native = loadNative();
 
   return native.getFinalCubestoreResult(transformDataJson, rows, resultData);
 };
 
-export const getFinalCubestoreResultArray = (transformDataJson: string[], rows: any[], resultDataJson: string[]): ArrayBuffer => {
+export const getFinalCubestoreResultArray = (transformDataJson: string[], rows: any[], resultDataJson: string[]): Promise<ArrayBuffer> => {
   const native = loadNative();
 
   return native.getFinalCubestoreResultArray(transformDataJson, rows, resultDataJson);
 };
 
-export const getFinalCubestoreResultMulti = (transformDataJson: string[], rows: any[], responseData: string): ArrayBuffer => {
+export const getFinalCubestoreResultMulti = (transformDataJson: string[], rows: any[], responseData: string): Promise<ArrayBuffer> => {
   const native = loadNative();
 
   return native.getFinalCubestoreResultMulti(transformDataJson, rows, responseData);