Skip to content

Commit e543cd0

Browse files
tzolovmarkpollack
authored andcommitted
Update AnthropicChatModel and Spring AI Documentation for Multimodal Support
- Enhance AnthropicChatModel to support PDF and document content types - Introduce getContentBlockTypeByMedia method for flexible media type handling - Update ContentBlock handling to dynamically determine content type for media - Add multimodal PDF support test case for Claude 3.5 Sonnet - Update documentation to reflect PDF and multimodal capabilities - Modify comparison chart to show PDF support for Anthropic Claude Fixese #1819 review
1 parent 9d207e3 commit e543cd0

File tree

8 files changed

+390
-309
lines changed

8 files changed

+390
-309
lines changed

models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.springframework.ai.anthropic.api.AnthropicApi.ChatCompletionRequest;
3838
import org.springframework.ai.anthropic.api.AnthropicApi.ChatCompletionResponse;
3939
import org.springframework.ai.anthropic.api.AnthropicApi.ContentBlock;
40+
import org.springframework.ai.anthropic.api.AnthropicApi.ContentBlock.Source;
4041
import org.springframework.ai.anthropic.api.AnthropicApi.ContentBlock.Type;
4142
import org.springframework.ai.anthropic.api.AnthropicApi.Role;
4243
import org.springframework.ai.anthropic.metadata.AnthropicUsage;
@@ -58,6 +59,7 @@
5859
import org.springframework.ai.chat.prompt.ChatOptions;
5960
import org.springframework.ai.chat.prompt.ChatOptionsBuilder;
6061
import org.springframework.ai.chat.prompt.Prompt;
62+
import org.springframework.ai.model.Media;
6163
import org.springframework.ai.model.ModelOptionsUtils;
6264
import org.springframework.ai.model.function.FunctionCallback;
6365
import org.springframework.ai.model.function.FunctionCallbackContext;
@@ -355,6 +357,18 @@ else if (mediaData instanceof String text) {
355357

356358
}
357359

360+
private Type getContentBlockTypeByMedia(Media media) {
361+
String mimeType = media.getMimeType().toString();
362+
if (mimeType.startsWith("image")) {
363+
return Type.IMAGE;
364+
}
365+
else if (mimeType.contains("pdf")) {
366+
return Type.DOCUMENT;
367+
}
368+
throw new IllegalArgumentException("Unsupported media type: " + mimeType
369+
+ ". Supported types are: images (image/*) and PDF documents (application/pdf)");
370+
}
371+
358372
ChatCompletionRequest createRequest(Prompt prompt, boolean stream) {
359373

360374
Set<String> functionsForThisRequest = new HashSet<>();
@@ -367,11 +381,12 @@ ChatCompletionRequest createRequest(Prompt prompt, boolean stream) {
367381
List<ContentBlock> contents = new ArrayList<>(List.of(new ContentBlock(message.getContent())));
368382
if (message instanceof UserMessage userMessage) {
369383
if (!CollectionUtils.isEmpty(userMessage.getMedia())) {
370-
List<ContentBlock> mediaContent = userMessage.getMedia()
371-
.stream()
372-
.map(media -> new ContentBlock(media.getMimeType().toString(),
373-
this.fromMediaData(media.getData())))
374-
.toList();
384+
List<ContentBlock> mediaContent = userMessage.getMedia().stream().map(media -> {
385+
Type contentBlockType = getContentBlockTypeByMedia(media);
386+
var source = new Source(media.getMimeType().toString(),
387+
this.fromMediaData(media.getData()));
388+
return new ContentBlock(contentBlockType, source);
389+
}).toList();
375390
contents.addAll(mediaContent);
376391
}
377392
}

0 commit comments

Comments
 (0)