perf(schema-compiler): Support native YAML transpilation (#9948)

* Prepare transpileYaml in native + schema compiler

* fix test

fix/sync error message

* fix test

Author: KSDaemon

packages/cubejs-backend-native/js/index.ts

@@ -517,6 +517,16 @@ export const transpileJs = async (transpileRequests: TransformConfig[]): Promise
   throw new Error('TranspileJs native implementation not found!');
 };
 
+export const transpileYaml = async (transpileRequests: TransformConfig[]): Promise<TransformResponse[]> => {
+  const native = loadNative();
+
+  if (native.transpileYaml) {
+    return native.transpileYaml(transpileRequests);
+  }
+
+  throw new Error('TranspileYaml native implementation not found!');
+};
+
 export interface PyConfiguration {
   repositoryFactory?: (ctx: unknown) => Promise<unknown>,
   logger?: (msg: string, params: Record<string, any>) => void,

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -13,7 +13,7 @@ import workerpool from 'workerpool';
 import { LRUCache } from 'lru-cache';
 
 import { FileContent, getEnv, isNativeSupported, SchemaFileRepository } from '@cubejs-backend/shared';
-import { NativeInstance, PythonCtx, transpileJs } from '@cubejs-backend/native';
+import { NativeInstance, PythonCtx, transpileJs, transpileYaml } from '@cubejs-backend/native';
 import { UserError } from './UserError';
 import { ErrorReporter, ErrorReporterOptions, SyntaxErrorInterface } from './ErrorReporter';
 import { CONTEXT_SYMBOLS, CubeDefinition, CubeSymbols } from './CubeSymbols';
@@ -707,7 +707,7 @@ export class DataSchemaCompiler {
   private async transpileYamlFile(
     file: FileContent,
     errorsReport: ErrorReporter,
-    { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId, stage }: TranspileOptions
+    { cubeNames, cubeSymbols, compilerId }: TranspileOptions
   ): Promise<(FileContent | undefined)> {
     const cacheKey = crypto.createHash('md5').update(JSON.stringify(file.content)).digest('hex');
 
@@ -717,9 +717,24 @@ export class DataSchemaCompiler {
       return { ...file, content };
     }
 
-    /* if (getEnv('transpilationNative')) {
+    if (getEnv('transpilationNative')) {
+      const reqData = {
+        fileName: file.fileName,
+        fileContent: file.content,
+        transpilers: [],
+        compilerId: compilerId || '',
+      };
+
+      errorsReport.inFile(file);
+      const res = await transpileYaml([reqData]);
+      errorsReport.addErrors(res[0].errors);
+      errorsReport.addWarnings(res[0].warnings as unknown as SyntaxErrorInterface[]);
+      errorsReport.exitFile();
+
+      this.compiledYamlCache.set(cacheKey, res[0].code);
 
-    } else */ if (getEnv('transpilationWorkerThreads')) {
+      return { ...file, content: res[0].code };
+    } else if (getEnv('transpilationWorkerThreads')) {
       const data = {
         fileName: file.fileName,
         content: file.content,

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -277,7 +277,7 @@ export class YamlCompiler {
       const pythonParser = new PythonParser(codeString);
       return pythonParser.transpileToJs();
     } catch (e: any) {
-      errorsReport.error(`Can't parse python expression. Most likely this type of syntax isn't supported yet: ${e.message || e}`);
+      errorsReport.error(`Failed to parse Python expression. Most likely this type of syntax isn't supported yet: ${e.message || e}`);
     }
 
     return t.nullLiteral();

packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml

@@ -62,7 +62,7 @@ cubes:
           allowAll: true
       - role: admin
         conditions:
-          - if: "{ !security_context.isBlocked }"
+          - if: "{ security_context.isNotBlocked }"
         rowLevel:
           filters:
             - member: status

packages/cubejs-schema-compiler/test/unit/fixtures/orders_incorrect_acl.yml

@@ -62,7 +62,7 @@ cubes:
           allowAll: true
       - role: admin
         conditions:
-          - if: "{ !security_context.isBlocked }"
+          - if: "{ security_context.isNotBlocked }"
         rowLevel:
           filters:
             - member: "{CUBE}.order_users.name"

packages/cubejs-schema-compiler/test/unit/fixtures/orders_nonexist_acl.yml

@@ -62,7 +62,7 @@ cubes:
           allowAll: true
       - role: admin
         conditions:
-          - if: "{ !security_context.isBlocked }"
+          - if: "{ security_context.isNotBlocked }"
         rowLevel:
           filters:
             - member: "{CUBE}.other.path.created_at"

packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts

@@ -119,7 +119,7 @@ describe('Yaml Schema Testing', () => {
     const { compiler } = prepareYamlCompiler(
       `cubes:
       - name: Products
-        sql: select { "string"+123 } from tbl
+        sql: select { "string"+123 } as a1, { 123abc } as a2 from tbl
         dimensions:
     `
     );
@@ -129,7 +129,7 @@ describe('Yaml Schema Testing', () => {
 
       throw new Error('compile must return an error');
     } catch (e: any) {
-      expect(e.message).toContain('Can\'t parse python expression');
+      expect(e.message).toContain('Failed to parse Python expression');
     }
   });
 
@@ -150,7 +150,7 @@ describe('Yaml Schema Testing', () => {
 
       throw new Error('compile must return an error');
     } catch (e: any) {
-      expect(e.message).toContain('name isn\'t defined for dimension: ');
+      expect(e.message).toContain('name isn\'t defined for dimension');
     }
   });
 
@@ -223,7 +223,9 @@ describe('Yaml Schema Testing', () => {
 
         throw new Error('compile must return an error');
       } catch (e: any) {
-        expect(e.message).toContain('Users cube: "title" must be a string');
+        expect(e.message).toMatch(
+          /Users cube: "title" (must be a string|is not allowed to be empty)/
+        );
       }
     });
 
@@ -479,7 +481,7 @@ describe('Yaml Schema Testing', () => {
         await compiler.compile();
         throw new Error('compile must return an error');
       } catch (e: any) {
-        expect(e.message).toContain('dimension.granularitys must be defined as array');
+        expect(e.message).toContain('must be defined as array');
       }
     });
 
@@ -544,7 +546,7 @@ describe('Yaml Schema Testing', () => {
           accessPolicy:
             - role: admin
               conditions:
-                - if: "{ !security_context.isBlocked }"
+                - if: "{ security_context.isNotBlocked }"
               rowLevel:
                 filters:
                   - member: status