AI: manually strip 5 backticks from LLM response in Insights agent

See bug for context, we sometimes see the LLM wrap its response in 5
backticks for no apparent reason. We're going to sync with the Aida
folks to see if we can understand why or fix this on their side, but for
now a fix is also to strip the 5 backticks if they are found in the
response.

Bug: 405054694
Change-Id: Ieb7499af142f7f7cba9ffea999729f49e1fbe536
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6375525
Auto-Submit: Jack Franklin <[email protected]>
Reviewed-by: Ergün Erdoğmuş <[email protected]>
Commit-Queue: Ergün Erdoğmuş <[email protected]>

Author: jackfranklin

front_end/models/ai_assistance/agents/PerformanceInsightsAgent.test.ts

@@ -37,6 +37,51 @@ describeWithEnvironment('PerformanceInsightsAgent', () => {
     assert.strictEqual(context.getTitle(), 'Insight: LCP by phase');
   });
 
+  // See b/405054694 for context on why we do this.
+  describe('parsing text responses', () => {
+    it('strips out 5 backticks if the response has them', async () => {
+      const agent = new PerformanceInsightsAgent({aidaClient: mockAidaClient()});
+      const response = agent.parseTextResponse('`````hello world`````');
+      assert.deepEqual(response, {answer: 'hello world'});
+    });
+
+    it('strips any newlines before the backticks', async () => {
+      const agent = new PerformanceInsightsAgent({aidaClient: mockAidaClient()});
+      const response = agent.parseTextResponse('\n\n`````hello world`````');
+      assert.deepEqual(response, {answer: 'hello world'});
+    });
+
+    it('does not strip the backticks if the response does not fully start and end with them', async () => {
+      const agent = new PerformanceInsightsAgent({aidaClient: mockAidaClient()});
+      const response = agent.parseTextResponse('answer: `````hello world`````');
+      assert.deepEqual(response, {answer: 'answer: `````hello world`````'});
+    });
+
+    it('does not strip the backticks in the middle of the response even if the response is also wrapped', async () => {
+      const agent = new PerformanceInsightsAgent({aidaClient: mockAidaClient()});
+      const response = agent.parseTextResponse('`````hello ````` world`````');
+      assert.deepEqual(response, {answer: 'hello ````` world'});
+    });
+
+    it('does not strip out inline code backticks', async () => {
+      const agent = new PerformanceInsightsAgent({aidaClient: mockAidaClient()});
+      const response = agent.parseTextResponse('This is code `console.log("hello")`');
+      assert.deepEqual(response, {answer: 'This is code `console.log("hello")`'});
+    });
+
+    it('does not strip out code block 3 backticks', async () => {
+      const agent = new PerformanceInsightsAgent({aidaClient: mockAidaClient()});
+      const response = agent.parseTextResponse(`\`\`\`
+code
+\`\`\``);
+      assert.deepEqual(response, {
+        answer: `\`\`\`
+code
+\`\`\``
+      });
+    });
+  });
+
   describe('handleContextDetails', () => {
     it('outputs the right context for the initial query from the user', async () => {
       const mockInsight = new TimelineUtils.InsightAIContext.ActiveInsight(FAKE_LCP_MODEL, FAKE_PARSED_TRACE);

front_end/models/ai_assistance/agents/PerformanceInsightsAgent.ts

@@ -17,9 +17,10 @@ import {
   type ContextDetail,
   type ContextResponse,
   ConversationContext,
+  type ParsedResponse,
   type RequestOptions,
   type ResponseData,
-  ResponseType,
+  ResponseType
 } from './AiAgent.js';
 
 const UIStringsNotTranslated = {
@@ -266,6 +267,23 @@ The fields are:
     });
   }
 
+  override parseTextResponse(response: string): ParsedResponse {
+    /**
+     * Sometimes the LLM responds with code chunks that wrap a text based markdown response.
+     * If this happens, we want to remove those before continuing.
+     * See b/405054694 for more details.
+     */
+    const trimmed = response.trim();
+    const FIVE_BACKTICKS = '`````';
+    if (trimmed.startsWith(FIVE_BACKTICKS) && trimmed.endsWith(FIVE_BACKTICKS)) {
+      // Purposefully use the trimmed text here; we might as well remove any
+      // newlines that are at the very start or end.
+      const stripped = trimmed.slice(FIVE_BACKTICKS.length, -FIVE_BACKTICKS.length);
+      return super.parseTextResponse(stripped);
+    }
+    return super.parseTextResponse(response);
+  }
+
   override async enhanceQuery(
       query: string,
       selectedInsight: ConversationContext<TimelineUtils.InsightAIContext.ActiveInsight>|null): Promise<string> {