[AiAssistance] Add screenshot image input to history

There is a storage limit (similar to how it is set and calculated in
front_end/panels/recorder/models/ScreenshotStorage.ts)

Pending: A placeholder image in case image is no longer available in history

Bug: 394029490
Change-Id: I797eb9aa77145585e51cbb4df17034f1bc21d15e
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6276245
Reviewed-by: Alex Rudenko <[email protected]>
Commit-Queue: Samiya Caur <[email protected]>

Author: Samiya Caur

front_end/panels/ai_assistance/AiAssistancePanel.test.ts

@@ -613,12 +613,11 @@ describeWithMockConnection('AI Assistance Panel', () => {
         contextMenu.invokeHandler(freestylerEntry.id());
       });
       assert.isTrue(view.lastCall.args[0].isReadOnly);
-      // Currently history should not store image input
       assert.deepEqual(view.lastCall.args[0].messages, [
         {
           entity: AiAssistance.ChatMessageEntity.USER,
           text: 'User question to Freestyler?',
-          imageInput: undefined,
+          imageInput,
         },
         {
           answer: 'test',

front_end/panels/ai_assistance/AiAssistancePanel.ts

@@ -278,7 +278,9 @@ export class AiAssistancePanel extends UI.Panel.Panel {
       accountFullName: syncInfo.accountFullName,
     };
 
-    this.#conversations = AiHistoryStorage.instance().getHistory().map(item => Conversation.fromSerialized(item));
+    this.#conversations = AiHistoryStorage.instance().getHistory().map(item => {
+      return new Conversation(item.type, item.history, item.id, true);
+    });
 
     if (isAiAssistancePatchingEnabled()) {
       // TODO: this is temporary code that should be replaced with workflow selection flow.
@@ -961,12 +963,14 @@ export class AiAssistancePanel extends UI.Panel.Panel {
       throw new Error('cross-origin context data should not be included');
     }
 
+    const image = isAiAssistanceMultimodalInputEnabled() ? imageInput : undefined;
+    const imageId = image ? crypto.randomUUID() : undefined;
     const runner = this.#currentAgent.run(
         text, {
           signal,
           selected: context,
         },
-        isAiAssistanceMultimodalInputEnabled() ? imageInput : undefined);
+        image, imageId);
     UI.ARIAUtils.alert(lockedString(UIStringsNotTranslate.answerLoading));
     await this.#doConversation(this.#saveResponsesToCurrentConversation(runner));
     UI.ARIAUtils.alert(lockedString(UIStringsNotTranslate.answerReady));

front_end/panels/ai_assistance/AiHistoryStorage.test.ts

@@ -22,6 +22,39 @@ describe('AiHistoryStorage', () => {
     type: AiAssistance.ConversationType.NETWORK,
     history: [],
   };
+  const agent4: AiAssistance.SerializedConversation = {
+    id: 'id4',
+    type: AiAssistance.ConversationType.STYLING,
+    history: [
+      {
+        type: AiAssistance.ResponseType.USER_QUERY,
+        query: 'text',
+        imageId: 'image-id1',
+        imageInput: undefined,
+      },
+      {
+        type: AiAssistance.ResponseType.ANSWER,
+        text: 'answer',
+        complete: true,
+      },
+      {
+        type: AiAssistance.ResponseType.USER_QUERY,
+        query: 'text',
+        imageId: 'image-id2',
+        imageInput: undefined,
+      },
+    ],
+  };
+  const serializedImage1: AiAssistance.SerializedImage = {
+    id: 'image-id1',
+    data: 'imageInput',
+    mimeType: 'image/jpeg',
+  };
+  const serializedImage2: AiAssistance.SerializedImage = {
+    id: 'image-id2',
+    data: 'imageInput',
+    mimeType: 'image/jpeg',
+  };
 
   beforeEach(() => {
     let data: Record<string, string> = {};
@@ -48,8 +81,8 @@ describe('AiHistoryStorage', () => {
     });
   });
 
-  function getStorage() {
-    return AiAssistance.AiHistoryStorage.instance(true);
+  function getStorage(maxStorageSize?: number) {
+    return AiAssistance.AiHistoryStorage.instance({forceNew: true, maxStorageSize});
   }
 
   it('should create and retrieve history entry', async () => {
@@ -141,6 +174,77 @@ describe('AiHistoryStorage', () => {
           },
         ],
     );
+    assert.deepEqual(
+        storage.getImageHistory(),
+        [],
+    );
+
+    await storage.upsertImage(serializedImage1);
+    await storage.upsertImage(serializedImage2);
+    await storage.upsertHistoryEntry(agent4);
+    assert.deepEqual(
+        storage.getHistory(),
+        [
+          {
+            id: 'id1',
+            type: 'freestyler' as AiAssistance.ConversationType,
+            history: [
+              {
+                type: AiAssistance.ResponseType.USER_QUERY,
+                query: 'text',
+              },
+            ],
+          },
+          {
+            id: 'id2',
+            type: 'drjones-file' as AiAssistance.ConversationType,
+            history: [],
+          },
+          {
+            id: 'id3',
+            type: 'drjones-network-request' as AiAssistance.ConversationType,
+            history: [],
+          },
+          {
+            id: 'id4',
+            type: 'freestyler' as AiAssistance.ConversationType,
+            history: [
+              {
+                type: AiAssistance.ResponseType.USER_QUERY,
+                query: 'text',
+                imageId: 'image-id1',
+                imageInput: undefined,
+              },
+              {
+                type: AiAssistance.ResponseType.ANSWER,
+                text: 'answer',
+                complete: true,
+              },
+              {
+                type: AiAssistance.ResponseType.USER_QUERY,
+                query: 'text',
+                imageId: 'image-id2',
+                imageInput: undefined,
+              },
+            ],
+          },
+        ],
+    );
+    assert.deepEqual(
+        storage.getImageHistory(),
+        [
+          {
+            id: 'image-id1',
+            data: 'imageInput',
+            mimeType: 'image/jpeg',
+          },
+          {
+            id: 'image-id2',
+            data: 'imageInput',
+            mimeType: 'image/jpeg',
+          }
+        ],
+    );
   });
 
   it('should delete a single entry', async () => {
@@ -155,9 +259,38 @@ describe('AiHistoryStorage', () => {
           {
             id: 'id1',
             type: 'freestyler' as AiAssistance.ConversationType,
-            history: [
+            history: [],
+          },
+          {
+            id: 'id3',
+            type: 'drjones-network-request' as AiAssistance.ConversationType,
+            history: [],
+          },
+        ],
+    );
+  });
 
-            ],
+  it('should delete image history entry', async () => {
+    const storage = getStorage();
+    await storage.upsertHistoryEntry(agent1);
+    await storage.upsertHistoryEntry(agent2);
+    await storage.upsertHistoryEntry(agent3);
+    await storage.upsertImage(serializedImage1);
+    await storage.upsertImage(serializedImage2);
+    await storage.upsertHistoryEntry(agent4);
+    await storage.deleteHistoryEntry('id4');
+    assert.deepEqual(
+        storage.getHistory(),
+        [
+          {
+            id: 'id1',
+            type: 'freestyler' as AiAssistance.ConversationType,
+            history: [],
+          },
+          {
+            id: 'id2',
+            type: 'drjones-file' as AiAssistance.ConversationType,
+            history: [],
           },
           {
             id: 'id3',
@@ -166,17 +299,74 @@ describe('AiHistoryStorage', () => {
           },
         ],
     );
+    assert.deepEqual(
+        storage.getImageHistory(),
+        [],
+    );
   });
 
   it('should delete all entries', async () => {
     const storage = getStorage();
     await storage.upsertHistoryEntry(agent1);
     await storage.upsertHistoryEntry(agent2);
     await storage.upsertHistoryEntry(agent3);
+    await storage.upsertImage(serializedImage1);
+    await storage.upsertImage(serializedImage2);
+    await storage.upsertHistoryEntry(agent4);
     await storage.deleteAll();
     assert.deepEqual(
         storage.getHistory(),
         [],
     );
+    assert.deepEqual(
+        storage.getImageHistory(),
+        [],
+    );
+  });
+
+  it('should limit the amount of stored images', async () => {
+    const storage = getStorage(2);
+
+    await storage.upsertImage({
+      id: 'image-id1',
+      data: '1',
+      mimeType: 'image/jpeg',
+    });
+    await storage.upsertHistoryEntry(agent1);
+    await storage.upsertImage({
+      id: 'image-id2',
+      data: '2',
+      mimeType: 'image/jpeg',
+    });
+    await storage.upsertImage({
+      id: 'image-id3',
+      data: '3',
+      mimeType: 'image/jpeg',
+    });
+    await storage.upsertHistoryEntry(agent2);
+    await storage.upsertImage({
+      id: 'image-id4',
+      data: '4',
+      mimeType: 'image/jpeg',
+    });
+    await storage.upsertHistoryEntry(agent3);
+    const imageHistory = storage.getImageHistory();
+    const imageData1 = imageHistory.find(item => item.id === 'image-id1');
+    const imageData2 = imageHistory.find(item => item.id === 'image-id2');
+    const imageData3 = imageHistory.find(item => item.id === 'image-id3');
+    const imageData4 = imageHistory.find(item => item.id === 'image-id4');
+
+    assert.notExists(imageData1);
+    assert.notExists(imageData2);
+    assert.deepEqual(imageData3, {
+      id: 'image-id3',
+      data: '3',
+      mimeType: 'image/jpeg',
+    });
+    assert.deepEqual(imageData4, {
+      id: 'image-id4',
+      data: '4',
+      mimeType: 'image/jpeg',
+    });
   });
 });