Skip to content

Commit a6ad0c8

Browse files
fix: add media/image support in Spring AI MessageConverter
Previously, MessageConverter only transferred text content from ADK to Spring AI, ignoring image and media attachments. This caused vision model requests to fail even though Spring AI's underlying models (like GPT-4o) support image inputs. Updated MessageConverter to properly handle image/media parts by constructing UserMessage with Media attachments. Fixes #705
1 parent e01df11 commit a6ad0c8

File tree

2 files changed

+149
-6
lines changed

2 files changed

+149
-6
lines changed

contrib/spring-ai/src/main/java/com/google/adk/models/springai/MessageConverter.java

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -243,11 +243,7 @@ private List<Message> handleUserContent(Content content) {
243243
}
244244

245245
List<Message> messages = new ArrayList<>();
246-
// Create UserMessage with text
247-
// TODO: Media attachments support - UserMessage constructors with media are private in Spring
248-
// AI 1.1.0
249-
// For now, only text content is supported
250-
messages.add(new UserMessage(textBuilder.toString()));
246+
messages.add(UserMessage.builder().text(textBuilder.toString()).media(mediaList).build());
251247
messages.addAll(toolResponseMessages);
252248

253249
return messages;

contrib/spring-ai/src/test/java/com/google/adk/models/springai/MessageConverterTest.java

Lines changed: 148 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@ void testToLlmPromptWithUserMessage() {
6060
assertThat(prompt.getInstructions()).hasSize(1);
6161
Message message = prompt.getInstructions().get(0);
6262
assertThat(message).isInstanceOf(UserMessage.class);
63-
assertThat(((UserMessage) message).getText()).isEqualTo("Hello, how are you?");
63+
UserMessage userMessage = (UserMessage) message;
64+
assertThat(userMessage.getText()).isEqualTo("Hello, how are you?");
65+
assertThat(userMessage.getMedia()).isEmpty();
6466
}
6567

6668
@Test
@@ -444,4 +446,149 @@ void testCombineMultipleSystemMessagesForGeminiCompatibility() {
444446
assertThat(secondMessage).isInstanceOf(UserMessage.class);
445447
assertThat(((UserMessage) secondMessage).getText()).isEqualTo("Hello world");
446448
}
449+
450+
@Test
451+
void testUserMessageWithInlineMediaData() {
452+
// Test conversion of ADK Content with inline media (image bytes) to Spring AI UserMessage
453+
byte[] imageData = "fake-image-data".getBytes();
454+
String mimeType = "image/png";
455+
456+
Content userContent =
457+
Content.builder()
458+
.role("user")
459+
.parts(
460+
List.of(
461+
Part.fromText("What's in this image?"),
462+
Part.builder()
463+
.inlineData(
464+
com.google.genai.types.Blob.builder()
465+
.mimeType(mimeType)
466+
.data(imageData)
467+
.build())
468+
.build()))
469+
.build();
470+
471+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
472+
473+
Prompt prompt = messageConverter.toLlmPrompt(request);
474+
475+
assertThat(prompt.getInstructions()).hasSize(1);
476+
Message message = prompt.getInstructions().get(0);
477+
assertThat(message).isInstanceOf(UserMessage.class);
478+
479+
UserMessage userMessage = (UserMessage) message;
480+
assertThat(userMessage.getText()).isEqualTo("What's in this image?");
481+
assertThat(userMessage.getMedia()).hasSize(1);
482+
org.springframework.ai.content.Media media = userMessage.getMedia().get(0);
483+
assertThat(media.getMimeType().toString()).isEqualTo(mimeType);
484+
assertThat(media.getData()).isInstanceOf(byte[].class);
485+
byte[] actualData = (byte[]) media.getData();
486+
assertThat(actualData).isEqualTo(imageData);
487+
}
488+
489+
@Test
490+
void testUserMessageWithFileMediaData() {
491+
// Test conversion of ADK Content with file-based media (URI) to Spring AI UserMessage
492+
String fileUri = "gs://bucket/image.jpg";
493+
String mimeType = "image/jpeg";
494+
495+
Content userContent =
496+
Content.builder()
497+
.role("user")
498+
.parts(
499+
List.of(
500+
Part.fromText("Analyze this image"),
501+
Part.builder()
502+
.fileData(
503+
com.google.genai.types.FileData.builder()
504+
.mimeType(mimeType)
505+
.fileUri(fileUri)
506+
.build())
507+
.build()))
508+
.build();
509+
510+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
511+
512+
Prompt prompt = messageConverter.toLlmPrompt(request);
513+
514+
assertThat(prompt.getInstructions()).hasSize(1);
515+
Message message = prompt.getInstructions().get(0);
516+
assertThat(message).isInstanceOf(UserMessage.class);
517+
518+
UserMessage userMessage = (UserMessage) message;
519+
assertThat(userMessage.getText()).isEqualTo("Analyze this image");
520+
assertThat(userMessage.getMedia()).hasSize(1);
521+
org.springframework.ai.content.Media media = userMessage.getMedia().get(0);
522+
assertThat(media.getMimeType().toString()).isEqualTo(mimeType);
523+
assertThat(media.getData()).isInstanceOf(String.class);
524+
String actualUri = (String) media.getData();
525+
assertThat(actualUri).isEqualTo(fileUri);
526+
}
527+
528+
@Test
529+
void testUserMessageWithMultipleMediaAttachments() {
530+
// Test conversion with multiple media attachments
531+
byte[] image1 = "image1-data".getBytes();
532+
byte[] image2 = "image2-data".getBytes();
533+
534+
Content userContent =
535+
Content.builder()
536+
.role("user")
537+
.parts(
538+
List.of(
539+
Part.fromText("Compare these images"),
540+
Part.builder()
541+
.inlineData(
542+
com.google.genai.types.Blob.builder()
543+
.mimeType("image/png")
544+
.data(image1)
545+
.build())
546+
.build(),
547+
Part.builder()
548+
.inlineData(
549+
com.google.genai.types.Blob.builder()
550+
.mimeType("image/jpeg")
551+
.data(image2)
552+
.build())
553+
.build()))
554+
.build();
555+
556+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
557+
558+
Prompt prompt = messageConverter.toLlmPrompt(request);
559+
560+
assertThat(prompt.getInstructions()).hasSize(1);
561+
UserMessage userMessage = (UserMessage) prompt.getInstructions().get(0);
562+
assertThat(userMessage.getText()).isEqualTo("Compare these images");
563+
assertThat(userMessage.getMedia()).hasSize(2);
564+
}
565+
566+
@Test
567+
void testUserMessageWithMediaOnly() {
568+
// Test conversion with media but no text
569+
byte[] imageData = "image-only".getBytes();
570+
571+
Content userContent =
572+
Content.builder()
573+
.role("user")
574+
.parts(
575+
List.of(
576+
Part.builder()
577+
.inlineData(
578+
com.google.genai.types.Blob.builder()
579+
.mimeType("image/png")
580+
.data(imageData)
581+
.build())
582+
.build()))
583+
.build();
584+
585+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
586+
587+
Prompt prompt = messageConverter.toLlmPrompt(request);
588+
589+
assertThat(prompt.getInstructions()).hasSize(1);
590+
UserMessage userMessage = (UserMessage) prompt.getInstructions().get(0);
591+
assertThat(userMessage.getText()).isEmpty();
592+
assertThat(userMessage.getMedia()).hasSize(1);
593+
}
447594
}

0 commit comments

Comments
 (0)