Skip to content

Make allow image url for Anthropic Message API #2825

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Make allow image url for Anthropic Message API #2825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions ...pring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
* @author Thomas Vitale
* @author Claudio Silva Junior
* @author Alexandros Pappas
* @author Jonghoon Park
* @since 1.0.0
*/
public class AnthropicChatModel implements ChatModel {
Expand Down Expand Up @@ -357,6 +358,18 @@ private ChatResponseMetadata from(AnthropicApi.ChatCompletionResponse result, Us
.build();
}

private Source getSourceByMedia(Media media) {
String data = this.fromMediaData(media.getData());

// http is not allowed and redirect not allowed
if (data.startsWith("https://")) {
return new Source(data);
}
else {
return new Source(media.getMimeType().toString(), data);
}
}

private String fromMediaData(Object mediaData) {
if (mediaData instanceof byte[] bytes) {
return Base64.getEncoder().encodeToString(bytes);
Expand Down Expand Up @@ -457,8 +470,7 @@ ChatCompletionRequest createRequest(Prompt prompt, boolean stream) {
if (!CollectionUtils.isEmpty(userMessage.getMedia())) {
List<ContentBlock> mediaContent = userMessage.getMedia().stream().map(media -> {
Type contentBlockType = getContentBlockTypeByMedia(media);
var source = new Source(media.getMimeType().toString(),
this.fromMediaData(media.getData()));
var source = getSourceByMedia(media);
return new ContentBlock(contentBlockType, source);
}).toList();
contents.addAll(mediaContent);
Expand Down
13 changes: 10 additions & 3 deletions .../spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/api/AnthropicApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023-2024 the original author or authors.
* Copyright 2023-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -57,6 +57,7 @@
* @author Thomas Vitale
* @author Jihoon Kim
* @author Alexandros Pappas
* @author Jonghoon Park
* @since 1.0.0
*/
public class AnthropicApi {
Expand Down Expand Up @@ -1038,13 +1039,15 @@ public String getValue() {
* @param mediaType The media type of the content. For example, "image/png" or
* "image/jpeg".
* @param data The base64-encoded data of the content.
* @param url The url of the content. (image only)
*/
@JsonInclude(Include.NON_NULL)
public record Source(
// @formatter:off
@JsonProperty("type") String type,
@JsonProperty("media_type") String mediaType,
@JsonProperty("data") String data) {
@JsonProperty("data") String data,
@JsonProperty("url") String url) {
// @formatter:on

/**
Expand All @@ -1053,7 +1056,11 @@ public record Source(
* @param data The content data.
*/
public Source(String mediaType, String data) {
this("base64", mediaType, data);
this("base64", mediaType, data, null);
}

public Source(String url) {
this("url", null, null, url);
}

}
Expand Down
8 changes: 3 additions & 5 deletions ...nthropic/src/test/java/org/springframework/ai/anthropic/client/AnthropicChatClientIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2023-2024 the original author or authors.
* Copyright 2023-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -301,14 +301,12 @@ void multiModalityEmbeddedImage(String modelName) throws IOException {
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Disabled("Currently Anthropic API does not support external image URLs")
@ParameterizedTest(name = "{0} : {displayName} ")
@ValueSource(strings = { "claude-3-opus-latest", "claude-3-5-sonnet-latest", "claude-3-haiku-latest",
"claude-3-7-sonnet-latest" })
@ValueSource(strings = { "claude-3-opus-latest", "claude-3-5-sonnet-latest", "claude-3-7-sonnet-latest" })
void multiModalityImageUrl(String modelName) throws IOException {

// TODO: add url method that wrapps the checked exception.
URL url = new URL("https://docs.spring.io/spring-ai/reference/1.0.0-SNAPSHOT/_images/multimodal.test.png");
URL url = new URL("https://docs.spring.io/spring-ai/reference/_images/multimodal.test.png");

// @formatter:off
String response = ChatClient.create(this.chatModel).prompt()
Expand Down