Skip to content

Commit b96de43

Browse files
Samiya CaurDevtools-frontend LUCI CQ
Samiya Caur
authored and
Devtools-frontend LUCI CQ
committed
[AiAssistance] Send screenshot as multimodal input to LLM
Also, updates to display the screenshot in chat (including when chat is reopened from history) Bug:394029490 Change-Id: If34f6b156ebf9b5b0a3c7ed955c9dbdee19be5ae Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6226459 Auto-Submit: Samiya Caur <[email protected]> Reviewed-by: Alex Rudenko <[email protected]> Commit-Queue: Samiya Caur <[email protected]>
1 parent be61c3b commit b96de43

File tree

11 files changed

+133
-20
lines changed

11 files changed

+133
-20
lines changed

‎front_end/panels/ai_assistance/AiAssistancePanel.test.ts‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
449449
{
450450
entity: AiAssistance.ChatMessageEntity.USER,
451451
text: 'test',
452+
imageInput: undefined,
452453
},
453454
{
454455
answer: 'test',
@@ -484,6 +485,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
484485
{
485486
entity: AiAssistance.ChatMessageEntity.USER,
486487
text: 'test',
488+
imageInput: undefined,
487489
},
488490
{
489491
answer: 'test',
@@ -520,6 +522,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
520522
{
521523
entity: AiAssistance.ChatMessageEntity.USER,
522524
text: 'test',
525+
imageInput: undefined,
523526
},
524527
{
525528
answer: 'test',
@@ -556,6 +559,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
556559
{
557560
entity: AiAssistance.ChatMessageEntity.USER,
558561
text: 'test',
562+
imageInput: undefined,
559563
},
560564
{
561565
answer: 'test',
@@ -573,15 +577,23 @@ describeWithMockConnection('AI Assistance Panel', () => {
573577
});
574578

575579
it('should switch agents and restore history', async () => {
580+
Object.assign(Root.Runtime.hostConfig, {
581+
devToolsFreestyler: {
582+
enabled: true,
583+
multimodal: true,
584+
},
585+
});
576586
const {view, panel} =
577587
createAiAssistancePanel({aidaClient: mockAidaClient([[{explanation: 'test'}], [{explanation: 'test2'}]])});
578588
panel.handleAction('freestyler.elements-floating-button');
579-
view.lastCall.args[0].onTextSubmit('User question to Freestyler?');
589+
const imageInput = {inlineData: {data: 'imageinputbytes', mimeType: 'image/jpeg'}};
590+
view.lastCall.args[0].onTextSubmit('User question to Freestyler?', imageInput);
580591
await drainMicroTasks();
581592
assert.deepEqual(view.lastCall.args[0].messages, [
582593
{
583594
entity: AiAssistance.ChatMessageEntity.USER,
584595
text: 'User question to Freestyler?',
596+
imageInput,
585597
},
586598
{
587599
answer: 'test',
@@ -599,6 +611,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
599611
{
600612
entity: AiAssistance.ChatMessageEntity.USER,
601613
text: 'User question to DrJones?',
614+
imageInput: undefined,
602615
},
603616
{
604617
answer: 'test2',
@@ -619,10 +632,12 @@ describeWithMockConnection('AI Assistance Panel', () => {
619632
contextMenu.invokeHandler(freestylerEntry.id());
620633

621634
await drainMicroTasks();
635+
// Currently history should not store image input
622636
assert.deepEqual(view.lastCall.args[0].messages, [
623637
{
624638
entity: AiAssistance.ChatMessageEntity.USER,
625639
text: 'User question to Freestyler?',
640+
imageInput: undefined,
626641
},
627642
{
628643
answer: 'test',
@@ -645,6 +660,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
645660
{
646661
entity: AiAssistance.ChatMessageEntity.USER,
647662
text: 'test',
663+
imageInput: undefined,
648664
},
649665
{
650666
answer: 'test',
@@ -678,6 +694,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
678694
{
679695
entity: AiAssistance.ChatMessageEntity.USER,
680696
text: 'test',
697+
imageInput: undefined,
681698
},
682699
{
683700
answer: 'test',
@@ -704,6 +721,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
704721
{
705722
entity: AiAssistance.ChatMessageEntity.USER,
706723
text: 'User question to Freestyler?',
724+
imageInput: undefined,
707725
},
708726
{
709727
answer: 'test',
@@ -721,6 +739,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
721739
{
722740
entity: AiAssistance.ChatMessageEntity.USER,
723741
text: 'User question to DrJones?',
742+
imageInput: undefined,
724743
},
725744
{
726745
answer: 'test2',
@@ -816,6 +835,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
816835
{
817836
entity: AiAssistance.ChatMessageEntity.USER,
818837
text: 'test',
838+
imageInput: undefined,
819839
},
820840
{
821841
answer: 'test',
@@ -839,6 +859,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
839859
{
840860
entity: AiAssistance.ChatMessageEntity.USER,
841861
text: 'test',
862+
imageInput: undefined,
842863
},
843864
{
844865
answer: 'test',
@@ -850,6 +871,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
850871
{
851872
entity: AiAssistance.ChatMessageEntity.USER,
852873
text: 'test2',
874+
imageInput: undefined,
853875
},
854876
{
855877
answer: 'test2',
@@ -1083,6 +1105,7 @@ describeWithMockConnection('AI Assistance Panel', () => {
10831105
{
10841106
entity: AiAssistance.ChatMessageEntity.USER,
10851107
text: 'test',
1108+
imageInput: undefined,
10861109
},
10871110
{
10881111
answer: undefined,
@@ -1151,5 +1174,40 @@ describeWithMockConnection('AI Assistance Panel', () => {
11511174

11521175
assert.isEmpty(view.lastCall.args[0].imageInput);
11531176
});
1177+
1178+
it('sends image as input', async () => {
1179+
Object.assign(Root.Runtime.hostConfig, {
1180+
devToolsFreestyler: {
1181+
enabled: true,
1182+
multimodal: true,
1183+
},
1184+
});
1185+
UI.Context.Context.instance().setFlavor(
1186+
ElementsPanel.ElementsPanel.ElementsPanel,
1187+
sinon.createStubInstance(ElementsPanel.ElementsPanel.ElementsPanel));
1188+
const {
1189+
view,
1190+
} = createAiAssistancePanel({aidaClient: mockAidaClient([[{explanation: 'test'}]])});
1191+
1192+
assert.isTrue(view.lastCall.args[0].multimodalInputEnabled);
1193+
1194+
view.lastCall.args[0].onTextSubmit('test', {inlineData: {data: 'imageInput', mimeType: 'image/jpeg'}});
1195+
await drainMicroTasks();
1196+
1197+
assert.deepEqual(view.lastCall.args[0].messages, [
1198+
{
1199+
entity: AiAssistance.ChatMessageEntity.USER,
1200+
text: 'test',
1201+
imageInput: {inlineData: {data: 'imageInput', mimeType: 'image/jpeg'}}
1202+
},
1203+
{
1204+
answer: 'test',
1205+
entity: AiAssistance.ChatMessageEntity.MODEL,
1206+
rpcId: undefined,
1207+
suggestions: undefined,
1208+
steps: [],
1209+
},
1210+
]);
1211+
});
11541212
});
11551213
});

‎front_end/panels/ai_assistance/AiAssistancePanel.ts‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -645,9 +645,9 @@ export class AiAssistancePanel extends UI.Panel.Panel {
645645
multimodalInputEnabled:
646646
isAiAssistanceMultimodalInputEnabled() && this.#currentAgent?.type === AgentType.STYLING,
647647
imageInput: this.#imageInput,
648-
onTextSubmit: (text: string) => {
648+
onTextSubmit: (text: string, imageInput?: Host.AidaClient.Part) => {
649649
this.#imageInput = '';
650-
void this.#startConversation(text);
650+
void this.#startConversation(text, imageInput);
651651
Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistanceQuerySubmitted);
652652
},
653653
onInspectElementClick: this.#handleSelectElementClick.bind(this),
@@ -920,7 +920,7 @@ export class AiAssistancePanel extends UI.Panel.Panel {
920920
return context;
921921
}
922922

923-
async #startConversation(text: string): Promise<void> {
923+
async #startConversation(text: string, imageInput?: Host.AidaClient.Part): Promise<void> {
924924
if (!this.#currentAgent) {
925925
return;
926926
}
@@ -935,10 +935,13 @@ export class AiAssistancePanel extends UI.Panel.Panel {
935935
// invariants do not hold anymore.
936936
throw new Error('cross-origin context data should not be included');
937937
}
938-
const runner = this.#currentAgent.run(text, {
939-
signal,
940-
selected: context,
941-
});
938+
939+
const runner = this.#currentAgent.run(
940+
text, {
941+
signal,
942+
selected: context,
943+
},
944+
isAiAssistanceMultimodalInputEnabled() ? imageInput : undefined);
942945
UI.ARIAUtils.alert(lockedString(UIStringsNotTranslate.answerLoading));
943946
await this.#doConversation(this.#saveResponsesToCurrentConversation(runner));
944947
UI.ARIAUtils.alert(lockedString(UIStringsNotTranslate.answerReady));
@@ -1011,6 +1014,7 @@ Output one filename per line and nothing else!
10111014
this.#messages.push({
10121015
entity: ChatMessageEntity.USER,
10131016
text: data.query,
1017+
imageInput: data.imageInput,
10141018
});
10151019
systemMessage = {
10161020
entity: ChatMessageEntity.MODEL,

‎front_end/panels/ai_assistance/AiHistoryStorage.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,12 @@ export class Conversation {
5252
}
5353

5454
addHistoryItem(item: ResponseData): void {
55-
this.history.push(item);
55+
if (item.type === ResponseType.USER_QUERY) {
56+
const historyItem = {...item, imageInput: undefined};
57+
this.history.push(historyItem);
58+
} else {
59+
this.history.push(item);
60+
}
5661
void AiHistoryStorage.instance().upsertHistoryEntry(this.serialize());
5762
}
5863

‎front_end/panels/ai_assistance/agents/AiAgent.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ describeWithEnvironment('AiAgent', () => {
224224
{
225225
type: ResponseType.USER_QUERY,
226226
query: 'query',
227+
imageInput: undefined,
227228
},
228229
{
229230
type: ResponseType.QUERYING,
@@ -279,6 +280,7 @@ describeWithEnvironment('AiAgent', () => {
279280
{
280281
type: ResponseType.USER_QUERY,
281282
query: 'query',
283+
imageInput: undefined,
282284
},
283285
{
284286
type: ResponseType.QUERYING,

‎front_end/panels/ai_assistance/agents/AiAgent.ts‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,13 @@ export interface ActionResponse {
8282
export interface QueryResponse {
8383
type: ResponseType.QUERYING;
8484
query?: string;
85+
imageInput?: Host.AidaClient.Part;
8586
}
8687

8788
export interface UserQuery {
8889
type: ResponseType.USER_QUERY;
8990
query: string;
91+
imageInput?: Host.AidaClient.Part;
9092
}
9193

9294
export type ResponseData = AnswerResponse|SuggestionsResponse|ErrorResponse|ActionResponse|SideEffectResponse|
@@ -247,10 +249,12 @@ export abstract class AiAgent<T> {
247249
return query;
248250
}
249251

250-
buildRequest(part: Host.AidaClient.Part, role: Host.AidaClient.Role.USER|Host.AidaClient.Role.ROLE_UNSPECIFIED):
251-
Host.AidaClient.AidaRequest {
252+
buildRequest(
253+
part: Host.AidaClient.Part|Host.AidaClient.Part[],
254+
role: Host.AidaClient.Role.USER|Host.AidaClient.Role.ROLE_UNSPECIFIED): Host.AidaClient.AidaRequest {
255+
const parts = Array.isArray(part) ? part : [part];
252256
const currentMessage: Host.AidaClient.Content = {
253-
parts: [part],
257+
parts,
254258
role,
255259
};
256260
const history = [...this.#history];
@@ -342,9 +346,11 @@ export abstract class AiAgent<T> {
342346
throw new Error('Unexpected action found');
343347
}
344348

345-
async * run(initialQuery: string, options: {
346-
signal?: AbortSignal, selected: ConversationContext<T>|null,
347-
}): AsyncGenerator<ResponseData, void, void> {
349+
async *
350+
run(initialQuery: string, options: {
351+
signal?: AbortSignal, selected: ConversationContext<T>|null,
352+
},
353+
imageInput?: Host.AidaClient.Part): AsyncGenerator<ResponseData, void, void> {
348354
await options.selected?.refresh();
349355

350356
// First context set on the agent determines its origin from now on.
@@ -360,13 +366,15 @@ export abstract class AiAgent<T> {
360366

361367
Host.userMetrics.freestylerQueryLength(enhancedQuery.length);
362368

363-
let query: Host.AidaClient.Part = {text: enhancedQuery};
369+
let query: Host.AidaClient.Part|Host.AidaClient.Part[];
370+
query = imageInput ? [{text: enhancedQuery}, imageInput] : [{text: enhancedQuery}];
364371
// Request is built here to capture history up to this point.
365372
let request = this.buildRequest(query, Host.AidaClient.Role.USER);
366373

367374
yield {
368375
type: ResponseType.USER_QUERY,
369376
query: initialQuery,
377+
imageInput,
370378
};
371379

372380
yield* this.handleContextDetails(options.selected);

‎front_end/panels/ai_assistance/agents/FileAgent.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ describeWithMockConnection('FileAgent', () => {
141141
{
142142
type: ResponseType.USER_QUERY,
143143
query: 'test',
144+
imageInput: undefined,
144145
},
145146
{
146147
type: ResponseType.CONTEXT,

‎front_end/panels/ai_assistance/agents/NetworkAgent.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ describeWithMockConnection('NetworkAgent', () => {
155155
{
156156
type: ResponseType.USER_QUERY,
157157
query: 'test',
158+
imageInput: undefined,
158159
},
159160
{
160161
type: ResponseType.CONTEXT,

‎front_end/panels/ai_assistance/agents/PerformanceAgent.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ self: 3
160160
{
161161
type: ResponseType.USER_QUERY,
162162
query: 'test',
163+
imageInput: undefined,
163164
},
164165
{
165166
type: ResponseType.CONTEXT,

‎front_end/panels/ai_assistance/agents/StylingAgent.test.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -774,6 +774,7 @@ STOP`,
774774
{
775775
type: AiAssistance.ResponseType.USER_QUERY,
776776
query: 'test',
777+
imageInput: undefined,
777778
},
778779
{
779780
type: AiAssistance.ResponseType.CONTEXT,
@@ -883,6 +884,7 @@ STOP`,
883884
{
884885
type: AiAssistance.ResponseType.USER_QUERY,
885886
query: 'test',
887+
imageInput: undefined,
886888
},
887889
{
888890
type: AiAssistance.ResponseType.CONTEXT,
@@ -930,6 +932,7 @@ STOP`,
930932
{
931933
type: AiAssistance.ResponseType.USER_QUERY,
932934
query: 'test',
935+
imageInput: undefined,
933936
},
934937
{
935938
type: AiAssistance.ResponseType.CONTEXT,
@@ -976,6 +979,7 @@ STOP`,
976979
{
977980
type: AiAssistance.ResponseType.USER_QUERY,
978981
query: 'test',
982+
imageInput: undefined,
979983
},
980984
{
981985
type: AiAssistance.ResponseType.CONTEXT,
@@ -1039,6 +1043,7 @@ STOP
10391043
{
10401044
type: AiAssistance.ResponseType.USER_QUERY,
10411045
query: 'test',
1046+
imageInput: undefined,
10421047
},
10431048
{
10441049
type: AiAssistance.ResponseType.CONTEXT,
@@ -1090,6 +1095,7 @@ STOP
10901095
{
10911096
type: AiAssistance.ResponseType.USER_QUERY,
10921097
query: 'test',
1098+
imageInput: undefined,
10931099
},
10941100
{
10951101
type: AiAssistance.ResponseType.CONTEXT,

0 commit comments

Comments
 (0)