introduce compiledJinjaCache

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -71,6 +71,7 @@ export type DataSchemaCompilerOptions = {
   allowNodeRequire?: boolean;
   compiledScriptCache: LRUCache<string, vm.Script>;
   compiledYamlCache: LRUCache<string, string>;
+  compiledJinjaCache: LRUCache<string, string>;
 };
 
 export type TranspileOptions = {
@@ -148,6 +149,8 @@ export class DataSchemaCompiler {
 
   private readonly compiledYamlCache: LRUCache<string, string>;
 
+  private readonly compiledJinjaCache: LRUCache<string, string>;
+
   private compileV8ContextCache: vm.Context | null = null;
 
   // FIXME: Is public only because of tests, should be private
@@ -182,6 +185,7 @@ export class DataSchemaCompiler {
     this.compilerId = options.compilerId || 'default';
     this.compiledScriptCache = options.compiledScriptCache;
     this.compiledYamlCache = options.compiledYamlCache;
+    this.compiledJinjaCache = options.compiledJinjaCache;
   }
 
   public compileObjects(compileServices: CompilerInterface[], objects, errorsReport: ErrorReporter) {
@@ -706,17 +710,28 @@ export class DataSchemaCompiler {
     errorsReport: ErrorReporter,
     options: TranspileOptions
   ): Promise<(FileContent | undefined)> {
-    const renderedFile = await this.yamlCompiler.renderTemplate(
-      file,
-      this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
-      this.pythonContext!
-    );
+    const cacheKey = crypto.createHash('md5').update(JSON.stringify(file.content)).digest('hex');
+
+    let renderedFileContent: string;
+
+    if (this.compiledJinjaCache.has(cacheKey)) {
+      renderedFileContent = this.compiledJinjaCache.get(cacheKey)!;
+    } else {
+      const renderedFile = await this.yamlCompiler.renderTemplate(
+        file,
+        this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
+        this.pythonContext!
+      );
+      renderedFileContent = renderedFile.content;
+
+      this.compiledJinjaCache.set(cacheKey, renderedFileContent);
+    }
 
     // We update the jinja/yaml file content to the rendered content
     // to reuse the same transpileYamlFile() flow afterward which
     // will update the content to the transpiled js content
     // avoiding costly YAML/Python parsing again.
-    file.content = renderedFile.content;
+    file.content = renderedFileContent;
 
     return this.transpileYamlFile(file, errorsReport, options);
   }

packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts

@@ -38,6 +38,7 @@ export type PrepareCompilerOptions = {
   adapter?: string;
   compiledScriptCache?: LRUCache<string, vm.Script>;
   compiledYamlCache?: LRUCache<string, string>;
+  compiledJinjaCache?: LRUCache<string, string>;
 };
 
 export interface CompilerInterface {
@@ -61,6 +62,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
 
   const compiledScriptCache = options.compiledScriptCache || new LRUCache<string, vm.Script>({ max: 250 });
   const compiledYamlCache = options.compiledYamlCache || new LRUCache<string, string>({ max: 250 });
+  const compiledJinjaCache = options.compiledJinjaCache || new LRUCache<string, string>({ max: 250 });
 
   const transpilers: TranspilerInterface[] = [
     new ValidationTranspiler(),
@@ -82,6 +84,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     viewCompilationGate,
     compiledScriptCache,
     compiledYamlCache,
+    compiledJinjaCache,
     viewCompilers: [viewCompiler],
     cubeCompilers: [cubeEvaluator, joinGraph, metaTransformer],
     contextCompilers: [contextEvaluator],