introduce compiledJinjaCache

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -70,6 +70,7 @@ export type DataSchemaCompilerOptions = {
   compileContext?: any;
   allowNodeRequire?: boolean;
   compiledScriptCache: LRUCache<string, vm.Script>;
+  compiledJinjaCache: LRUCache<string, string>;
 };
 
 export type TranspileOptions = {
@@ -145,6 +146,8 @@ export class DataSchemaCompiler {
 
   private readonly compiledScriptCache: LRUCache<string, vm.Script>;
 
+  private readonly compiledJinjaCache: LRUCache<string, string>;
+
   private compileV8ContextCache: vm.Context | null = null;
 
   // FIXME: Is public only because of tests, should be private
@@ -178,6 +181,7 @@ export class DataSchemaCompiler {
     this.workerPool = null;
     this.compilerId = options.compilerId || 'default';
     this.compiledScriptCache = options.compiledScriptCache;
+    this.compiledJinjaCache = options.compiledJinjaCache;
   }
 
   public compileObjects(compileServices: CompilerInterface[], objects, errorsReport: ErrorReporter) {
@@ -684,17 +688,28 @@ export class DataSchemaCompiler {
     errorsReport: ErrorReporter,
     options: TranspileOptions
   ): Promise<(FileContent | undefined)> {
-    const renderedFile = await this.yamlCompiler.renderTemplate(
-      file,
-      this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
-      this.pythonContext!
-    );
+    const cacheKey = crypto.createHash('md5').update(JSON.stringify(file.content)).digest('hex');
+
+    let renderedFileContent: string;
+
+    if (this.compiledJinjaCache.has(cacheKey)) {
+      renderedFileContent = this.compiledJinjaCache.get(cacheKey)!;
+    } else {
+      const renderedFile = await this.yamlCompiler.renderTemplate(
+        file,
+        this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
+        this.pythonContext!
+      );
+      renderedFileContent = renderedFile.content;
+
+      this.compiledJinjaCache.set(cacheKey, renderedFileContent);
+    }
 
     // We update the jinja/yaml file content to the rendered content
     // to reuse the same transpileYamlFile() flow afterward which
     // will update the content to the transpiled js content
     // avoiding costly YAML/Python parsing again.
-    file.content = renderedFile.content;
+    file.content = renderedFileContent;
 
     return this.transpileYamlFile(file, errorsReport, options);
   }

packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts

@@ -37,6 +37,7 @@ export type PrepareCompilerOptions = {
   headCommitId?: string;
   adapter?: string;
   compiledScriptCache?: LRUCache<string, vm.Script>;
+  compiledJinjaCache?: LRUCache<string, string>;
 };
 
 export interface CompilerInterface {
@@ -59,6 +60,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
   const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler);
 
   const compiledScriptCache = options.compiledScriptCache || new LRUCache<string, vm.Script>({ max: 250 });
+  const compiledJinjaCache = options.compiledJinjaCache || new LRUCache<string, string>({ max: 250 });
 
   const transpilers: TranspilerInterface[] = [
     new ValidationTranspiler(),
@@ -79,6 +81,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     transpilers,
     viewCompilationGate,
     compiledScriptCache,
+    compiledJinjaCache,
     viewCompilers: [viewCompiler],
     cubeCompilers: [cubeEvaluator, joinGraph, metaTransformer],
     contextCompilers: [contextEvaluator],