[AiAssistance] Display placeholder image when image data unavailable in history

ergunsh · Devtools-frontend LUCI CQ · commit e0032e49131c · 2025-03-04T05:16:13.000-08:00
Bug: 396620504 Change-Id: I6043e317c57ed69232925c66819f66353e260827 Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6275378 Auto-Submit: Samiya Caur <samiyac@chromium.org> Commit-Queue: Alex Rudenko <alexrudenko@chromium.org> Commit-Queue: Ergün Erdoğmuş <ergunsh@chromium.org> Reviewed-by: Alex Rudenko <alexrudenko@chromium.org>
diff --git a/front_end/panels/ai_assistance/AiAssistancePanel.ts b/front_end/panels/ai_assistance/AiAssistancePanel.ts
@@ -1270,7 +1270,7 @@ export class AiAssistancePanel extends UI.Panel.Panel {
     for await (const data of items) {
       // We don't want to save partial responses to the conversation history.
       if (data.type !== ResponseType.ANSWER || data.complete) {
-        currentConversation?.addHistoryItem(data);
+        void currentConversation?.addHistoryItem(data);
       }
       yield data;
     }
diff --git a/front_end/panels/ai_assistance/AiHistoryStorage.test.ts b/front_end/panels/ai_assistance/AiHistoryStorage.test.ts
@@ -325,7 +325,8 @@ describe('AiHistoryStorage', () => {
   });
 
   it('should limit the amount of stored images', async () => {
-    const storage = getStorage(2);
+    const MAX_STORAGE_SIZE = 2;
+    const storage = getStorage(MAX_STORAGE_SIZE);
 
     await storage.upsertImage({
       id: 'image-id1',
@@ -402,5 +403,115 @@ describe('AiHistoryStorage', () => {
                'this is more than 80 characters because I\'m just going to keep typing words and …');
          });
     });
+
+    describe('addHistoryItem', () => {
+      const historyItem1: AiAssistance.ResponseData = {
+        type: AiAssistance.ResponseType.USER_QUERY,
+        query: 'text',
+        imageInput: {
+          inlineData: {
+            data: '1',
+            mimeType: 'image/jpeg',
+          }
+        },
+        imageId: 'image-id1',
+      };
+      const historyItem2: AiAssistance.ResponseData = {
+        type: AiAssistance.ResponseType.USER_QUERY,
+        query: 'text',
+        imageInput: {
+          inlineData: {
+            data: '2',
+            mimeType: 'image/jpeg',
+          }
+        },
+        imageId: 'image-id2',
+      };
+
+      it('should store images and text conversation separately', async () => {
+        const storage = getStorage();
+        sinon.stub(AiAssistance.AiHistoryStorage, 'instance').returns(storage);
+        const conversation1 = new AiAssistance.Conversation(AiAssistance.ConversationType.STYLING, [], 'id1', false);
+        await conversation1.addHistoryItem(historyItem1);
+        const conversation2 = new AiAssistance.Conversation(AiAssistance.ConversationType.STYLING, [], 'id2', false);
+        await conversation2.addHistoryItem(historyItem2);
+
+        const imageHistory = storage.getImageHistory();
+        assert.lengthOf(imageHistory, 2);
+        assert.deepEqual(imageHistory[0], {
+          id: 'image-id1',
+          data: '1',
+          mimeType: 'image/jpeg',
+        });
+        assert.deepEqual(imageHistory[1], {
+          id: 'image-id2',
+          data: '2',
+          mimeType: 'image/jpeg',
+        });
+
+        const historyWithoutImages = storage.getHistory();
+        assert.lengthOf(historyWithoutImages, 2);
+        assert.deepEqual(historyWithoutImages[0], {
+          id: 'id1',
+          type: AiAssistance.ConversationType.STYLING,
+          history: [{
+            type: AiAssistance.ResponseType.USER_QUERY,
+            query: 'text',
+            imageId: 'image-id1',
+          }]
+        });
+        assert.deepEqual(historyWithoutImages[1], {
+          id: 'id2',
+          type: AiAssistance.ConversationType.STYLING,
+          history: [{
+            type: AiAssistance.ResponseType.USER_QUERY,
+            query: 'text',
+            imageInput: undefined,
+            imageId: 'image-id2',
+          }]
+        });
+      });
+
+      it('should have empty image data for image not present in history', async () => {
+        const MAX_STORAGE_SIZE = 1;
+        const storage = getStorage(MAX_STORAGE_SIZE);
+        sinon.stub(AiAssistance.AiHistoryStorage, 'instance').returns(storage);
+        const conversation1 = new AiAssistance.Conversation(AiAssistance.ConversationType.STYLING, [], 'id1', false);
+        await conversation1.addHistoryItem(historyItem1);
+        const conversation2 = new AiAssistance.Conversation(AiAssistance.ConversationType.STYLING, [], 'id2', false);
+        await conversation2.addHistoryItem(historyItem2);
+
+        const imageHistory = storage.getImageHistory();
+        assert.lengthOf(imageHistory, 1);
+        const historyWithoutImages = storage.getHistory();
+        assert.lengthOf(historyWithoutImages, 2);
+        const conversationFromHistory = historyWithoutImages.map(item => {
+          return new AiAssistance.Conversation(item.type, item.history, item.id, true);
+        });
+        assert.lengthOf(conversationFromHistory, 2);
+        assert.deepEqual(conversationFromHistory[0].history, [{
+                           type: AiAssistance.ResponseType.USER_QUERY,
+                           query: 'text',
+                           imageInput: {
+                             inlineData: {
+                               data: AiAssistance.NOT_FOUND_IMAGE_DATA,
+                               mimeType: 'image/jpeg',
+                             }
+                           },
+                           imageId: 'image-id1',
+                         }]);
+        assert.deepEqual(conversationFromHistory[1].history, [{
+                           type: AiAssistance.ResponseType.USER_QUERY,
+                           query: 'text',
+                           imageInput: {
+                             inlineData: {
+                               data: '2',
+                               mimeType: 'image/jpeg',
+                             }
+                           },
+                           imageId: 'image-id2',
+                         }]);
+      });
+    });
   });
 });
diff --git a/front_end/panels/ai_assistance/AiHistoryStorage.ts b/front_end/panels/ai_assistance/AiHistoryStorage.ts
@@ -17,6 +17,8 @@ export const enum ConversationType {
   PERFORMANCE_INSIGHT = 'performance-insight',
 }
 
+export const NOT_FOUND_IMAGE_DATA = '';
+
 export interface SerializedConversation {
   id: string;
   type: ConversationType;
@@ -71,7 +73,8 @@ export class Conversation {
       for (const data of historyWithoutImages) {
         if (data.type === ResponseType.USER_QUERY && data.imageId) {
           const image = imageHistory.find(item => item.id === data.imageId);
-          const inlineData = image ? {data: image.data, mimeType: image.mimeType} : {data: '', mimeType: 'image/jpeg'};
+          const inlineData = image ? {data: image.data, mimeType: image.mimeType} :
+                                     {data: NOT_FOUND_IMAGE_DATA, mimeType: 'image/jpeg'};
           history.push({...data, imageInput: {inlineData}});
         } else {
           history.push(data);
@@ -86,16 +89,16 @@ export class Conversation {
     this.#isReadOnly = true;
   }
 
-  addHistoryItem(item: ResponseData): void {
+  async addHistoryItem(item: ResponseData): Promise<void> {
     if (item.type === ResponseType.USER_QUERY) {
       if (item.imageId && item.imageInput && 'inlineData' in item.imageInput) {
         const inlineData = item.imageInput.inlineData;
-        void AiHistoryStorage.instance().upsertImage(
+        await AiHistoryStorage.instance().upsertImage(
             {id: item.imageId, data: inlineData.data, mimeType: inlineData.mimeType});
       }
     }
     this.history.push(item);
-    void AiHistoryStorage.instance().upsertHistoryEntry(this.serialize());
+    await AiHistoryStorage.instance().upsertHistoryEntry(this.serialize());
   }
 
   serialize(): SerializedConversation {
diff --git a/front_end/panels/ai_assistance/ai_assistance-meta.ts b/front_end/panels/ai_assistance/ai_assistance-meta.ts
@@ -153,6 +153,15 @@ Common.Settings.registerSettingExtension({
   condition: isAnyFeatureAvailable,
 });
 
+Common.Settings.registerSettingExtension({
+  category: Common.Settings.SettingCategory.NONE,
+  settingName: 'ai-assistance-history-images',
+  settingType: Common.Settings.SettingType.ARRAY,
+  title: i18nLazyString(UIStrings.enableAiAssistance),
+  defaultValue: [],
+  condition: isAnyFeatureAvailable,
+});
+
 UI.ActionRegistration.registerActionExtension({
   actionId: 'freestyler.elements-floating-button',
   contextTypes(): [] {
diff --git a/front_end/panels/ai_assistance/components/ChatView.ts b/front_end/panels/ai_assistance/components/ChatView.ts
@@ -16,7 +16,7 @@ import * as UI from '../../../ui/legacy/legacy.js';
 import * as Lit from '../../../ui/lit/lit.js';
 import * as VisualLogging from '../../../ui/visual_logging/visual_logging.js';
 import {type ContextDetail, type ConversationContext, ErrorType} from '../agents/AiAgent.js';
-import {ConversationType} from '../AiHistoryStorage.js';
+import {ConversationType, NOT_FOUND_IMAGE_DATA} from '../AiHistoryStorage.js';
 import {PatchWidget} from '../PatchWidget.js';
 
 import stylesRaw from './chatView.css.js';
@@ -198,6 +198,10 @@ const UIStringsNotTranslate = {
    *@description Title for the x-link which wraps the image input rendered in chat messages.
    */
   openImageInNewTab: 'Open image in a new tab',
+  /**
+   *@description Alt text for image when it is not available.
+   */
+  imageUnavailable: 'Image unavailable',
   /**
    *@description Button text to change the selected workspace
    */
@@ -807,18 +811,9 @@ function renderChatMessage({
         html`<devtools-icon
           .name=${'profile'}
         ></devtools-icon>`;
-    let imageInput = html``;
-    if (message.imageInput && 'inlineData' in message.imageInput) {
-      const imageUrl = `data:image/jpeg;base64,${message.imageInput.inlineData.data}`;
-      // clang-format off
-      imageInput = html`<x-link
-        class="image-link" title=${UIStringsNotTranslate.openImageInNewTab}
-        href=${imageUrl}
-      >
-        <img src=${imageUrl} alt=${UIStringsNotTranslate.imageInputSentToTheModel} />
-      </x-link>`;
-      // clang-format on
-    }
+    const imageInput = message.imageInput && 'inlineData' in message.imageInput ?
+        renderImageChatMessage(message.imageInput.inlineData) :
+        Lit.nothing;
     // clang-format off
     return html`<section
       class="chat-message query"
@@ -884,6 +879,25 @@ function renderChatMessage({
   // clang-format on
 }
 
+function renderImageChatMessage(inlineData: Host.AidaClient.MediaBlob): Lit.LitTemplate {
+  if (inlineData.data === NOT_FOUND_IMAGE_DATA) {
+    // clang-format off
+    return html`<div class="unavailable-image" title=${UIStringsNotTranslate.imageUnavailable}>
+      <devtools-icon name='file-image'></devtools-icon>
+    </div>`;
+    // clang-format on
+  }
+  const imageUrl = `data:image/jpeg;base64,${inlineData.data}`;
+  // clang-format off
+    return html`<x-link
+      class="image-link" title=${UIStringsNotTranslate.openImageInNewTab}
+      href=${imageUrl}
+    >
+      <img src=${imageUrl} alt=${UIStringsNotTranslate.imageInputSentToTheModel} />
+    </x-link>`;
+  // clang-format on
+}
+
 function renderSelection({
   selectedContext,
   inspectElementToggled,
diff --git a/front_end/panels/ai_assistance/components/chatView.css b/front_end/panels/ai_assistance/components/chatView.css
@@ -322,6 +322,22 @@
       vertical-align: bottom;
     }
   }
+
+  .unavailable-image {
+    margin: var(--sys-size-4) 0;
+    display: inline-flex;
+    justify-content: center;
+    align-items: center;
+    height: var(--sys-size-17);
+    width: var(--sys-size-18);
+    background-color: var(--sys-color-surface3);
+    border-radius: var(--sys-shape-corner-small);
+    border: 1px solid var(--sys-color-neutral-outline);
+
+    devtools-icon {
+      color: var(--sys-color-state-disabled);
+    }
+  }
 }
 
 .select-element {

Original file line number	Diff line number	Diff line change
`@@ -1270,7 +1270,7 @@ export class AiAssistancePanel extends UI.Panel.Panel {`
`1270`	`1270`	`for await (const data of items) {`
`1271`	`1271`	`// We don't want to save partial responses to the conversation history.`
`1272`	`1272`	`if (data.type !== ResponseType.ANSWER \|\| data.complete) {`
`1273`		`- currentConversation?.addHistoryItem(data);`
	`1273`	`+ void currentConversation?.addHistoryItem(data);`
`1274`	`1274`	`}`
`1275`	`1275`	`yield data;`
`1276`	`1276`	`}`