more perf tracking

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -227,6 +227,8 @@ export class DataSchemaCompiler {
       ? files.filter(f => this.filesToCompile.includes(f.fileName)).map(f => ({ ...f }))
       : files.map(f => ({ ...f }));
 
+    const jinjaLoaderTimer = perfTracker.start('loadJinjaTemplates');
+
     const jinjaTemplatedFiles = toCompile.filter((file) => file.fileName.endsWith('.jinja') ||
       (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) && file.content.match(JINJA_SYNTAX));
 
@@ -235,6 +237,8 @@ export class DataSchemaCompiler {
       this.loadJinjaTemplates(jinjaTemplatedFiles);
     }
 
+    jinjaLoaderTimer.end();
+
     const errorsReport = new ErrorReporter(null, [], this.errorReportOptions);
     this.errorsReporter = errorsReport;
 
@@ -562,6 +566,8 @@ export class DataSchemaCompiler {
   ): Promise<(FileContent | undefined)> {
     try {
       if (getEnv('transpilationNative')) {
+        const compileJsFileTimer = perfTracker.start('transpileJsFile (native)');
+
         const reqData = {
           fileName: file.fileName,
           fileContent: file.content,
@@ -583,8 +589,12 @@ export class DataSchemaCompiler {
         errorsReport.addWarnings(res[0].warnings as unknown as SyntaxErrorInterface[]);
         errorsReport.exitFile();
 
+        compileJsFileTimer.end();
+
         return { ...file, content: res[0].code };
       } else if (getEnv('transpilationWorkerThreads')) {
+        const compileJsFileTimer = perfTracker.start('transpileJsFile (threads)');
+
         const data = {
           fileName: file.fileName,
           content: file.content,
@@ -597,8 +607,12 @@ export class DataSchemaCompiler {
         errorsReport.addErrors(res.errors);
         errorsReport.addWarnings(res.warnings);
 
+        compileJsFileTimer.end();
+
         return { ...file, content: res.content };
       } else {
+        const compileJsFileTimer = perfTracker.start('transpileJsFile (inplace)');
+
         const ast = parse(
           file.content,
           {
@@ -615,6 +629,9 @@ export class DataSchemaCompiler {
         errorsReport.exitFile();
 
         const content = babelGenerator(ast, {}, file.content).code;
+
+        compileJsFileTimer.end();
+
         return { ...file, content };
       }
     } catch (e: any) {
@@ -648,6 +665,8 @@ export class DataSchemaCompiler {
     }
 
     if (getEnv('transpilationNative')) {
+      const transpileYamlFileTimer = perfTracker.start('transpileYamlFile (native)');
+
       const reqData = {
         fileName: file.fileName,
         fileContent: file.content,
@@ -666,8 +685,12 @@ export class DataSchemaCompiler {
 
       this.compiledYamlCache.set(cacheKey, res[0].code);
 
+      transpileYamlFileTimer.end();
+
       return { ...file };
     } else if (getEnv('transpilationWorkerThreads')) {
+      const transpileYamlFileTimer = perfTracker.start('transpileYamlFile (threads)');
+
       const data = {
         fileName: file.fileName,
         content: file.content,
@@ -685,8 +708,12 @@ export class DataSchemaCompiler {
 
       this.compiledYamlCache.set(cacheKey, res.content);
 
+      transpileYamlFileTimer.end();
+
       return { ...file };
     } else {
+      const transpileYamlFileTimer = perfTracker.start('transpileYamlFile (inplace)');
+
       const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport);
 
       if (transpiledFile) {
@@ -696,6 +723,8 @@ export class DataSchemaCompiler {
 
       this.compiledYamlCache.set(cacheKey, transpiledFile?.content || '');
 
+      transpileYamlFileTimer.end();
+
       return transpiledFile;
     }
   }
@@ -710,6 +739,8 @@ export class DataSchemaCompiler {
     // } else if (getEnv('transpilationWorkerThreads')) {
     //
     // } else {
+    const transpileJinjaFileTimer = perfTracker.start('transpileJinjaFile (common)');
+
     const transpiledFile = await this.yamlCompiler.compileYamlWithJinjaFile(
       file,
       errorsReport,
@@ -724,6 +755,8 @@ export class DataSchemaCompiler {
       file.convertedToJs = true;
     }
 
+    transpileJinjaFileTimer.end();
+
     return transpiledFile;
     // }
   }

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -19,6 +19,7 @@ import { nonStringFields } from './CubeValidator';
 import { ErrorReporter } from './ErrorReporter';
 import { camelizeCube } from './utils';
 import { CompileContext } from './DataSchemaCompiler';
+import { perfTracker } from './PerfTracker';
 
 type EscapeStateStack = {
   inFormattedStr?: boolean;
@@ -72,7 +73,10 @@ export class YamlCompiler {
   ): Promise<FileContent | undefined> {
     const renderedFile = await this.renderTemplate(file, compileContext, pythonContext);
 
-    return this.transpileYamlFile(renderedFile, errorsReport);
+    const transpileJinjaFileTimer2 = perfTracker.start('compile Jinja - transpileYamlFile');
+    const res = this.transpileYamlFile(renderedFile, errorsReport);
+    transpileJinjaFileTimer2.end();
+    return res;
   }
 
   public transpileYamlFile(