more perf tracking

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -221,6 +221,8 @@ export class DataSchemaCompiler {
       ? files.filter(f => this.filesToCompile.includes(f.fileName))
       : files;
 
+    const jinjaLoaderTimer = perfTracker.start('loadJinjaTemplates');
+
     const jinjaTemplatedFiles = toCompile.filter((file) => file.fileName.endsWith('.jinja') ||
       (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) && file.content.match(JINJA_SYNTAX));
 
@@ -229,6 +231,8 @@ export class DataSchemaCompiler {
       this.loadJinjaTemplates(jinjaTemplatedFiles);
     }
 
+    jinjaLoaderTimer.end();
+
     const errorsReport = new ErrorReporter(null, [], this.errorReportOptions);
     this.errorsReporter = errorsReport;
 
@@ -555,6 +559,8 @@ export class DataSchemaCompiler {
   ): Promise<(FileContent | undefined)> {
     try {
       if (getEnv('transpilationNative')) {
+        const compileJsFileTimer = perfTracker.start('transpileJsFile (native)');
+
         const reqData = {
           fileName: file.fileName,
           fileContent: file.content,
@@ -576,8 +582,12 @@ export class DataSchemaCompiler {
         errorsReport.addWarnings(res[0].warnings as unknown as SyntaxErrorInterface[]);
         errorsReport.exitFile();
 
+        compileJsFileTimer.end();
+
         return { ...file, content: res[0].code };
       } else if (getEnv('transpilationWorkerThreads')) {
+        const compileJsFileTimer = perfTracker.start('transpileJsFile (threads)');
+
         const data = {
           fileName: file.fileName,
           content: file.content,
@@ -590,8 +600,12 @@ export class DataSchemaCompiler {
         errorsReport.addErrors(res.errors);
         errorsReport.addWarnings(res.warnings);
 
+        compileJsFileTimer.end();
+
         return { ...file, content: res.content };
       } else {
+        const compileJsFileTimer = perfTracker.start('transpileJsFile (inplace)');
+
         const ast = parse(
           file.content,
           {
@@ -608,6 +622,9 @@ export class DataSchemaCompiler {
         errorsReport.exitFile();
 
         const content = babelGenerator(ast, {}, file.content).code;
+
+        compileJsFileTimer.end();
+
         return { ...file, content };
       }
     } catch (e: any) {
@@ -632,6 +649,8 @@ export class DataSchemaCompiler {
     { cubeNames, cubeSymbols, compilerId }: TranspileOptions
   ): Promise<(FileContent | undefined)> {
     if (getEnv('transpilationNative')) {
+      const transpileYamlFileTimer = perfTracker.start('transpileYamlFile (native)');
+
       const reqData = {
         fileName: file.fileName,
         fileContent: file.content,
@@ -648,8 +667,12 @@ export class DataSchemaCompiler {
       file.content = res[0].code;
       file.convertedToJs = true;
 
+      transpileYamlFileTimer.end();
+
       return { ...file, content: res[0].code };
     } else if (getEnv('transpilationWorkerThreads')) {
+      const transpileYamlFileTimer = perfTracker.start('transpileYamlFile (threads)');
+
       const data = {
         fileName: file.fileName,
         content: file.content,
@@ -665,15 +688,21 @@ export class DataSchemaCompiler {
       file.content = res.content;
       file.convertedToJs = true;
 
+      transpileYamlFileTimer.end();
+
       return { ...file, content: res.content };
     } else {
+      const transpileYamlFileTimer = perfTracker.start('transpileYamlFile (inplace)');
+
       const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport);
 
       if (transpiledFile) {
         file.content = transpiledFile.content;
         file.convertedToJs = true;
       }
 
+      transpileYamlFileTimer.end();
+
       return transpiledFile;
     }
   }
@@ -688,6 +717,8 @@ export class DataSchemaCompiler {
     // } else if (getEnv('transpilationWorkerThreads')) {
     //
     // } else {
+    const transpileJinjaFileTimer = perfTracker.start('transpileJinjaFile (common)');
+
     const transpiledFile = await this.yamlCompiler.compileYamlWithJinjaFile(
       file,
       errorsReport,
@@ -702,6 +733,8 @@ export class DataSchemaCompiler {
       file.convertedToJs = true;
     }
 
+    transpileJinjaFileTimer.end();
+
     return transpiledFile;
     // }
   }

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -19,6 +19,7 @@ import { nonStringFields } from './CubeValidator';
 import { ErrorReporter } from './ErrorReporter';
 import { camelizeCube } from './utils';
 import { CompileContext } from './DataSchemaCompiler';
+import { perfTracker } from './PerfTracker';
 
 type EscapeStateStack = {
   inFormattedStr?: boolean;
@@ -72,7 +73,10 @@ export class YamlCompiler {
   ): Promise<FileContent | undefined> {
     const renderedFile = await this.renderTemplate(file, compileContext, pythonContext);
 
-    return this.transpileYamlFile(renderedFile, errorsReport);
+    const transpileJinjaFileTimer2 = perfTracker.start('compile Jinja - transpileYamlFile');
+    const res = this.transpileYamlFile(renderedFile, errorsReport);
+    transpileJinjaFileTimer2.end();
+    return res;
   }
 
   public transpileYamlFile(