Revert "remove perfTracker tracking..."

This reverts commit cf10a91.

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -23,6 +23,7 @@ import { CompilerInterface } from './PrepareCompiler';
 import { YamlCompiler } from './YamlCompiler';
 import { CubeDictionary } from './CubeDictionary';
 import { CompilerCache } from './CompilerCache';
+import { perfTracker } from './PerfTracker';
 
 const ctxFileStorage = new AsyncLocalStorage<FileContent>();
 
@@ -211,8 +212,12 @@ export class DataSchemaCompiler {
   }
 
   protected async doCompile() {
+    const compileTimer = perfTracker.start('doCompile', true);
+
     const files = await this.repository.dataSchemaFiles();
 
+    console.log(`Compiling ${files.length} files...`);
+
     this.pythonContext = await this.loadPythonContext(files, 'globals.py');
     this.yamlCompiler.initFromPythonContext(this.pythonContext);
 
@@ -235,6 +240,8 @@ export class DataSchemaCompiler {
     }
 
     const transpile = async (stage: CompileStage): Promise<FileContent[]> => {
+      const transpileTimer = perfTracker.start(`transpilation-stage-${stage}`);
+
       let cubeNames: string[] = [];
       let cubeSymbols: Record<string, Record<string, boolean>> = {};
       let transpilerNames: string[] = [];
@@ -301,6 +308,8 @@ export class DataSchemaCompiler {
         results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {})));
       }
 
+      transpileTimer.end();
+
       return results.filter(f => !!f) as FileContent[];
     };
 
@@ -398,6 +407,8 @@ export class DataSchemaCompiler {
     });
 
     const compilePhase = async (compilers: CompileCubeFilesCompilers, stage: 0 | 1 | 2 | 3) => {
+      const compilePhaseTimer = perfTracker.start(`compilation-phase-${stage}`);
+
       // clear the objects for the next phase
       cubes = [];
       exports = {};
@@ -406,7 +417,9 @@ export class DataSchemaCompiler {
       asyncModules = [];
       transpiledFiles = await transpile(stage);
 
-      return this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
+      const res = this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
+      compilePhaseTimer.end();
+      return res;
     };
 
     return compilePhase({ cubeCompilers: this.cubeNameCompilers }, 0)
@@ -442,6 +455,12 @@ export class DataSchemaCompiler {
         } else if (transpilationWorkerThreads && this.workerPool) {
           this.workerPool.terminate();
         }
+
+        // End overall compilation timing and print performance report
+        compileTimer.end();
+        setImmediate(() => {
+          perfTracker.printReport();
+        });
       });
   }
 
@@ -640,7 +659,9 @@ export class DataSchemaCompiler {
           asyncModules
         );
       });
+    const asyncModulesTimer = perfTracker.start('asyncModules.reduce (jinja)');
     await asyncModules.reduce((a: Promise<void>, b: CallableFunction) => a.then(() => b()), Promise.resolve());
+    asyncModulesTimer.end();
     return this.compileObjects(compilers.cubeCompilers || [], cubes, errorsReport)
       .then(() => this.compileObjects(compilers.contextCompilers || [], contexts, errorsReport));
   }
@@ -663,7 +684,9 @@ export class DataSchemaCompiler {
     compiledFiles[file.fileName] = true;
 
     if (file.fileName.endsWith('.js')) {
+      const compileJsFileTimer = perfTracker.start('compileJsFile');
       this.compileJsFile(file, errorsReport, { doSyntaxCheck });
+      compileJsFileTimer.end();
     } else if (file.fileName.endsWith('.yml.jinja') || file.fileName.endsWith('.yaml.jinja') ||
       (
         file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')
@@ -677,7 +700,9 @@ export class DataSchemaCompiler {
         this.pythonContext!
       ));
     } else if (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) {
+      const compileYamlFileTimer = perfTracker.start('compileYamlFile');
       this.yamlCompiler.compileYamlFile(file, errorsReport);
+      compileYamlFileTimer.end();
     }
   }
 

packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts

@@ -16,6 +16,7 @@ import { nonStringFields } from './CubeValidator';
 import { CubeDictionary } from './CubeDictionary';
 import { ErrorReporter } from './ErrorReporter';
 import { camelizeCube } from './utils';
+import { perfTracker } from './PerfTracker';
 
 type EscapeStateStack = {
   inFormattedStr?: boolean;
@@ -92,7 +93,9 @@ export class YamlCompiler {
     for (const key of Object.keys(yamlObj)) {
       if (key === 'cubes') {
         (yamlObj.cubes || []).forEach(({ name, ...cube }) => {
+          const transpileAndPrepareJsFileTimer = perfTracker.start('yaml-transpileAndPrepareJsFile');
           const transpiledFile = this.transpileAndPrepareJsFile(file, 'cube', { name, ...cube }, errorsReport);
+          transpileAndPrepareJsFileTimer.end();
           this.dataSchemaCompiler?.compileJsFile(transpiledFile, errorsReport);
         });
       } else if (key === 'views') {
@@ -134,7 +137,10 @@ export class YamlCompiler {
 
     cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport);
 
-    return this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport);
+    const transpileYamlTimer = perfTracker.start('transpileYaml');
+    const res = this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport);
+    transpileYamlTimer.end();
+    return res;
   }
 
   private transpileYaml(obj, propertyPath, cubeName, errorsReport: ErrorReporter) {
@@ -163,7 +169,9 @@ export class YamlCompiler {
               if (propertyPath[propertyPath.length - 1] === 'values') {
                 if (typeof code === 'string') {
                   if (code.match(PY_TEMPLATE_SYNTAX)) {
+                    const parsePythonAndTranspileToJsTimer184 = perfTracker.start('parsePythonAndTranspileToJs call 184');
                     ast = this.parsePythonAndTranspileToJs(`f"${this.escapeDoubleQuotes(code)}"`, errorsReport);
+                    parsePythonAndTranspileToJsTimer184.end();
                   } else {
                     ast = t.stringLiteral(code);
                   }
@@ -178,7 +186,9 @@ export class YamlCompiler {
                 }
               }
               if (ast === null) {
+                const parsePythonAndTranspileToJsTimer201 = perfTracker.start('parsePythonAndTranspileToJs call 201');
                 ast = this.parsePythonAndTranspileToJs(code, errorsReport);
+                parsePythonAndTranspileToJsTimer201.end();
               }
               return this.extractProgramBodyIfNeeded(ast);
             }).filter(ast => !!ast)))]);
@@ -189,7 +199,9 @@ export class YamlCompiler {
     }
 
     if (propertyPath[propertyPath.length - 1] === 'extends') {
+      const parsePythonAndTranspileToJsTimer214 = perfTracker.start('parsePythonAndTranspileToJs call 214');
       const ast = this.parsePythonAndTranspileToJs(obj, errorsReport);
+      parsePythonAndTranspileToJsTimer214.end();
       return this.astIntoArrowFunction(ast, obj, cubeName, name => this.cubeDictionary.resolveCube(name));
     } else if (typeof obj === 'string') {
       let code = obj;
@@ -203,7 +215,9 @@ export class YamlCompiler {
           code = `f"${this.escapeDoubleQuotes(obj)}"`;
         }
 
+        const parsePythonAndTranspileToJsTimer225 = perfTracker.start('parsePythonAndTranspileToJs call 225');
         const ast = this.parsePythonAndTranspileToJs(code, errorsReport);
+        parsePythonAndTranspileToJsTimer225.end();
         return this.extractProgramBodyIfNeeded(ast);
       }
 
@@ -288,7 +302,9 @@ export class YamlCompiler {
   }
 
   private parsePythonIntoArrowFunction(codeString: string, cubeName, originalObj, errorsReport: ErrorReporter) {
+    const parsePythonAndTranspileToJsTimer301 = perfTracker.start('parsePythonAndTranspileToJs call 301');
     const ast = this.parsePythonAndTranspileToJs(codeString, errorsReport);
+    parsePythonAndTranspileToJsTimer301.end();
     return this.astIntoArrowFunction(ast as any, codeString, cubeName);
   }
 
@@ -298,8 +314,12 @@ export class YamlCompiler {
     }
 
     try {
+      const parsePythonAndTranspileToJsTimer = perfTracker.start('PythonParser->transpileToJs()');
+
       const pythonParser = new PythonParser(codeString);
-      return pythonParser.transpileToJs();
+      const res = pythonParser.transpileToJs();
+      parsePythonAndTranspileToJsTimer.end();
+      return res;
     } catch (e: any) {
       errorsReport.error(`Can't parse python expression. Most likely this type of syntax isn't supported yet: ${e.message || e}`);
     }
@@ -308,6 +328,7 @@ export class YamlCompiler {
   }
 
   private astIntoArrowFunction(input: t.Program | t.NullLiteral, codeString: string, cubeName, resolveSymbol?: (string) => any) {
+    const astIntoArrowFunctionTimer = perfTracker.start('astIntoArrowFunction');
     const initialJs = babelGenerator(input, {}, codeString).code;
 
     // Re-parse generated JS to set all necessary parent paths
@@ -332,6 +353,7 @@ export class YamlCompiler {
     babelTraverse(ast, traverseObj);
 
     const body: any = ast.program.body[0];
+    astIntoArrowFunctionTimer.end();
     return body?.expression;
   }