fix: context files in prompts not working

runner/configuration/environment.ts

@@ -7,10 +7,7 @@ import {
   RootPromptDefinition,
 } from '../shared-interfaces.js';
 import { Rating } from '../ratings/rating-types.js';
-import {
-  extractPromptContextFilePatterns,
-  renderHandlebarsTemplate,
-} from './prompt-templating.js';
+import { renderHandlebarsTemplate } from './prompt-templating.js';
 import { McpServerOptions } from '../codegen/llm-runner.js';
 import { lazy } from '../utils/lazy-creation.js';
 import { EnvironmentConfig } from './environment-config.js';
@@ -120,21 +117,21 @@ export class Environment {
   }
 
   systemPromptGeneration = lazy(() => {
-    return this.renderRelativePrompt(this.config.generationSystemPrompt);
+    return this.renderRelativePrompt(this.config.generationSystemPrompt).result;
   });
 
   systemPromptRepair = lazy(() => {
     if (!this.config.repairSystemPrompt) {
       return 'Please fix the given errors and return the corrected code.';
     }
-    return this.renderRelativePrompt(this.config.repairSystemPrompt);
+    return this.renderRelativePrompt(this.config.repairSystemPrompt).result;
   });
 
   systemPromptEditing = lazy(() => {
     if (!this.config.editingSystemPrompt) {
       return this.systemPromptGeneration();
     }
-    return this.renderRelativePrompt(this.config.editingSystemPrompt);
+    return this.renderRelativePrompt(this.config.editingSystemPrompt).result;
   });
 
   /**
@@ -275,14 +272,12 @@ export class Environment {
     ratings: Rating[],
     isEditing: boolean
   ): PromptDefinition {
-    const { promptText, contextFiles } = extractPromptContextFilePatterns(
-      this.renderRelativePrompt(relativePath)
-    );
+    const { result, contextFiles } = this.renderRelativePrompt(relativePath);
 
     return {
       name: name,
       kind: 'single',
-      prompt: promptText,
+      prompt: result,
       ratings,
       systemPromptType: isEditing ? 'editing' : 'generation',
       contextFilePatterns: contextFiles,
@@ -387,7 +382,7 @@ export class Environment {
   }
 
   /** Renders a prompt from a path relative to the environment config. */
-  private renderRelativePrompt(relativePath: string): string {
+  private renderRelativePrompt(relativePath: string) {
     const path = resolve(this.rootPath, relativePath);
     return this.renderPrompt(readFileSync(path, 'utf8'), path);
   }

runner/configuration/prompt-templating.ts

@@ -35,21 +35,10 @@ initializeHandlebars();
 export function renderHandlebarsTemplate<T extends { rootDir: string | null }>(
   content: string,
   ctx: T
-): string {
+) {
   const template = Handlebars.compile(content, { strict: true });
-  return template(ctx);
-}
-
-/**
- * Extracts the context file patterns from a prompt's text. Returns the prompt's text without
- * any special context file syntax and the context file patterns.
- * @param initialPromptText Initial text of the prompt.
- */
-export function extractPromptContextFilePatterns(initialPromptText: string) {
   const contextFiles: string[] = [];
-  const promptText = Handlebars.compile(initialPromptText, {
-    strict: true,
-  })(null, {
+  const result = template(ctx, {
     partials: {
       contextFiles: (ctx) => {
         if (typeof ctx !== 'string') {
@@ -84,7 +73,7 @@ export function extractPromptContextFilePatterns(initialPromptText: string) {
   });
 
   return {
-    promptText,
+    result,
     contextFiles,
   };
 }

runner/ratings/autoraters/code-rater.ts

@@ -64,7 +64,7 @@ export async function autoRateCode(
   const prompt = environment.renderPrompt(promptText, null, {
     APP_PROMPT: appPrompt,
     FRAMEWORK_SPECIFIC_HINTS: FW_HINTS[environment.fullStackFramework.id] ?? '',
-  });
+  }).result;
 
   const result = await llm.generateConstrained({
     abortSignal,

runner/ratings/autoraters/visuals-rater.ts

@@ -31,7 +31,7 @@ export async function autoRateAppearance(
 ): Promise<AutoRateResult> {
   const prompt = environment.renderPrompt(defaultVisualRaterPrompt, null, {
     APP_PROMPT: appPrompt,
-  });
+  }).result;
 
   const messages: PromptDataMessage[] = [
     {