perf(schema-compiler): Improve yaml compilation speed (#9926)

* add convertedToJs flag

* pass toCompile to compileCubeFiles()

* refactored compileYamlFile() to not call compileJsFile directly

* moved yaml transpilation to transpile phase

* removed dataSchemaCompiler from YamlCompiler's deps

* renaming

* simplified code

* add PerfTracker

* inject perTracker measures in dataschemacompiler

* update CubePropContextTranspiler to support string keys

* replace original content with js on first phase

* update arrow func expression instead of blind copying

+ correct path collection

* fix grammar

* more types

* Preload Jinja templates to the engine

* remove perf logging

* simplified compileFile

* avoid mutating original repository files

* linter fix

* filter out files we don't transpile/compile

* move PerfTracker to backend-shared

* Refactor doCompile() to avoid content mutation

* refactor: split first and later compile/transpile stages

* improve transpilationNative

Author: KSDaemon

packages/cubejs-backend-shared/src/PerfTracker.ts

@@ -0,0 +1,88 @@
+import { performance, PerformanceObserver } from 'perf_hooks';
+
+interface PerfMetric {
+  count: number;
+  totalTime: number;
+  avgTime: number;
+}
+
+interface PerfStats {
+  [key: string]: PerfMetric;
+}
+
+class PerfTracker {
+  private metrics: PerfStats = {};
+
+  private globalMetric: string | null = null;
+
+  public constructor() {
+    const obs = new PerformanceObserver((items) => {
+      for (const entry of items.getEntries()) {
+        const { name } = entry;
+        if (!this.metrics[name]) {
+          this.metrics[name] = { count: 0, totalTime: 0, avgTime: 0 };
+        }
+        const m = this.metrics[name];
+        m.count++;
+        m.totalTime += entry.duration;
+        m.avgTime = m.totalTime / m.count;
+      }
+    });
+    obs.observe({ entryTypes: ['measure'] });
+  }
+
+  public start(name: string, global: boolean = false): { end: () => void } {
+    const uid = `${name}-${performance.now()}`;
+    const startMark = `${uid}-start`;
+    const endMark = `${uid}-end`;
+    performance.mark(startMark);
+
+    if (global && !this.globalMetric) {
+      this.globalMetric = name;
+    }
+
+    let ended = false;
+
+    return {
+      end: () => {
+        if (ended) return;
+        performance.mark(endMark);
+        performance.measure(name, startMark, endMark);
+        ended = true;
+      }
+    };
+  }
+
+  public printReport() {
+    console.log('\n🚀 PERFORMANCE REPORT 🚀\n');
+    console.log('═'.repeat(90));
+
+    const sorted = Object.entries(this.metrics)
+      .sort(([, a], [, b]) => b.totalTime - a.totalTime);
+
+    if (!sorted.length) {
+      console.log('No performance data collected.');
+      return;
+    }
+
+    let totalTime: number = 0;
+
+    if (this.globalMetric) {
+      totalTime = this.metrics[this.globalMetric]?.totalTime;
+    } else {
+      totalTime = sorted.reduce((sum, [, m]) => sum + m.totalTime, 0);
+    }
+
+    console.log(`⏱️  TOTAL TIME: ${totalTime.toFixed(2)}ms\n`);
+
+    sorted.forEach(([name, m]) => {
+      const pct = totalTime > 0 ? (m.totalTime / totalTime * 100) : 0;
+      console.log(`  ${name.padEnd(40)} │ ${m.totalTime.toFixed(2).padStart(8)}ms │ ${m.avgTime.toFixed(2).padStart(7)}ms avg │ ${pct.toFixed(1).padStart(5)}% │ ${m.count.toString().padStart(4)} calls`);
+    });
+
+    console.log('═'.repeat(90));
+    console.log('🎯 End of Performance Report\n');
+  }
+}
+
+export const perfTracker = new PerfTracker();

packages/cubejs-backend-shared/src/index.ts

@@ -21,3 +21,4 @@ export * from './process';
 export * from './platform';
 export * from './FileRepository';
 export * from './decorators';
+export * from './PerfTracker';