diff --git a/docs/guides/ORCHESTRATION_CHAT_COMPLETION.md b/docs/guides/ORCHESTRATION_CHAT_COMPLETION.md index a3393e1f8..867c8cd83 100644 --- a/docs/guides/ORCHESTRATION_CHAT_COMPLETION.md +++ b/docs/guides/ORCHESTRATION_CHAT_COMPLETION.md @@ -259,6 +259,47 @@ try (Stream stream = client.streamChatCompletion(prompt, config)) { Please find [an example in our Spring Boot application](../../sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java). It shows the usage of Spring Boot's `ResponseBodyEmitter` to stream the chat completion delta messages to the frontend in real-time. + +## Add images and multiple text inputs to a message + +It's possible to add images and multiple text inputs to a message. + +### Add images to a message + +An image can be added to a message as follows. + +```java +var message = Message.user("Describe the following image"); +var newMessage = message.withImage("https://url.to/image.jpg"); +``` + +You can also construct a message with an image directly, using the `ImageItem` class. + +```java +var message = Message.user(new ImageItem("https://url.to/image.jpg")); +``` + +Some AI models, like GPT 4o, support additionally setting the detail level with which the image is read. This can be set via the `DetailLevel` parameter. + +```java +var newMessage = message.withImage("https://url.to/image.jpg", ImageItem.DetailLevel.LOW); +``` +Note, that currently only user messages are supported for image attachments. + +### Add multiple text inputs to a message + +It's also possible to add multiple text inputs to a message. This can be useful for providing additional context to the AI model. You can add additional text inputs as follows. + +```java +var message = Message.user("What is chess about?"); +var newMessage = message.withText("Answer in two sentences."); +``` + +Note, that only user and system messages are supported for multiple text inputs. + +Please find [an example in our Spring Boot application](../../sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java). + + ## Set model parameters Change your LLM configuration to add model parameters: diff --git a/docs/release-notes/release_notes.md b/docs/release-notes/release_notes.md index 3cd699b57..0dd4cd905 100644 --- a/docs/release-notes/release_notes.md +++ b/docs/release-notes/release_notes.md @@ -8,12 +8,14 @@ ### 🔧 Compatibility Notes -- +- `Message.content()` returns a `ContentItem` now instead of a `String`. Use `((TextItem) Message.content().items().get(0)).text()` if the corresponding `ContentItem` is a `TextItem` and the string representation is needed. ### ✨ New Functionality - Upgrade to release 2502a of AI Core. -- [Add Orchestration `LlamaGuardFilter`](../guides/ORCHESTRATION_CHAT_COMPLETION.md#chat-completion-filter). +- Orchestration: + - [Add `LlamaGuardFilter`](../guides/ORCHESTRATION_CHAT_COMPLETION.md#chat-completion-filter). + - [Convenient methods to create messages containing images and multiple text inputs](../guides/ORCHESTRATION_CHAT_COMPLETION.md#add-images-and-multiple-text-inputs-to-a-message) ### 📈 Improvements diff --git a/orchestration/pom.xml b/orchestration/pom.xml index c53117a45..7c6988682 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -31,11 +31,11 @@ ${project.basedir}/../ - 77% + 80% 92% - 92% - 70% - 92% + 93% + 71% + 95% 100% diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/AssistantMessage.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/AssistantMessage.java index f78178673..f9cb82d08 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/AssistantMessage.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/AssistantMessage.java @@ -1,6 +1,9 @@ package com.sap.ai.sdk.orchestration; +import com.google.common.annotations.Beta; +import java.util.List; import javax.annotation.Nonnull; +import lombok.Getter; import lombok.Value; import lombok.experimental.Accessors; @@ -13,5 +16,16 @@ public class AssistantMessage implements Message { @Nonnull String role = "assistant"; /** The content of the message. */ - @Nonnull String content; + @Nonnull + @Getter(onMethod_ = @Beta) + MessageContent content; + + /** + * Creates a new assistant message with the given single message. + * + * @param singleMessage the single message. + */ + public AssistantMessage(@Nonnull final String singleMessage) { + content = new MessageContent(List.of(new TextItem(singleMessage))); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ContentItem.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ContentItem.java new file mode 100644 index 000000000..8e77b6061 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ContentItem.java @@ -0,0 +1,8 @@ +package com.sap.ai.sdk.orchestration; + +/** + * Represents an item in a {@link MessageContent} object. + * + * @since 1.3.0 + */ +public sealed interface ContentItem permits TextItem, ImageItem {} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ImageItem.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ImageItem.java new file mode 100644 index 000000000..dfec057b0 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ImageItem.java @@ -0,0 +1,68 @@ +package com.sap.ai.sdk.orchestration; + +import java.util.Locale; +import javax.annotation.Nonnull; + +/** + * Represents an image item in a {@link MessageContent} object. + * + * @param imageUrl the URL of the image + * @param detailLevel the detail level of the image (optional) + * @since 1.3.0 + */ +public record ImageItem(@Nonnull String imageUrl, @Nonnull DetailLevel detailLevel) + implements ContentItem { + + /** + * Creates a new image item with the given image URL. + * + * @param imageUrl the URL of the image + * @since 1.3.0 + */ + public ImageItem(@Nonnull final String imageUrl) { + this(imageUrl, DetailLevel.AUTO); + } + + /** + * The detail level of the image. + * + * @since 1.3.0 + */ + public enum DetailLevel { + /** Low detail level. */ + LOW("low"), + /** High detail level. */ + HIGH("high"), + /** Automatic detail level. */ + AUTO("auto"); + + private final String level; + + /** + * Converts a string to a detail level. + * + * @param str the string to convert + * @return the detail level + * @since 1.3.0 + */ + @Nonnull + static DetailLevel fromString(@Nonnull final String str) { + return DetailLevel.valueOf(str.toUpperCase(Locale.ENGLISH)); + } + + /** + * Get the string representation of the DetailLevel + * + * @return the DetailLevel as string + * @since 1.3.0 + */ + @Nonnull + public String toString() { + return level; + } + + DetailLevel(@Nonnull final String level) { + this.level = level; + } + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java index e3199dd01..30a0b53ad 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java @@ -2,43 +2,62 @@ import com.google.common.annotations.Beta; import com.sap.ai.sdk.orchestration.model.ChatMessage; +import com.sap.ai.sdk.orchestration.model.ImageContent; +import com.sap.ai.sdk.orchestration.model.ImageContentImageUrl; +import com.sap.ai.sdk.orchestration.model.MultiChatMessage; +import com.sap.ai.sdk.orchestration.model.MultiChatMessageContent; import com.sap.ai.sdk.orchestration.model.SingleChatMessage; +import com.sap.ai.sdk.orchestration.model.TextContent; +import java.util.LinkedList; +import java.util.List; import javax.annotation.Nonnull; /** Interface representing convenience wrappers of chat message to the orchestration service. */ public sealed interface Message permits UserMessage, AssistantMessage, SystemMessage { /** - * A convenience method to create a user message. + * A convenience method to create a user message from a string. * - * @param msg the message content. + * @param message the message content. * @return the user message. */ @Nonnull - static UserMessage user(@Nonnull final String msg) { - return new UserMessage(msg); + static UserMessage user(@Nonnull final String message) { + return new UserMessage(message); + } + + /** + * A convenience method to create a user message containing only an image. + * + * @param imageItem the message content. + * @return the user message. + * @since 1.3.0 + */ + @Nonnull + static UserMessage user(@Nonnull final ImageItem imageItem) { + return new UserMessage(new MessageContent(List.of(imageItem))); } /** * A convenience method to create an assistant message. * - * @param msg the message content. + * @param message the message content. * @return the assistant message. */ @Nonnull - static AssistantMessage assistant(@Nonnull final String msg) { - return new AssistantMessage(msg); + static AssistantMessage assistant(@Nonnull final String message) { + return new AssistantMessage(message); } /** - * A convenience method to create a system message. + * A convenience method to create a system message from a string. * - * @param msg the message content. + * @param message the message content. * @return the system message. */ @Nonnull - static SystemMessage system(@Nonnull final String msg) { - return new SystemMessage(msg); + static SystemMessage system(@Nonnull final String message) { + return new SystemMessage(message); } /** @@ -48,7 +67,21 @@ static SystemMessage system(@Nonnull final String msg) { */ @Nonnull default ChatMessage createChatMessage() { - return SingleChatMessage.create().role(role()).content(content()); + final var itemList = this.content().items(); + if (itemList.size() == 1 && itemList.get(0) instanceof TextItem textItem) { + return SingleChatMessage.create().role(role()).content(textItem.text()); + } + final var contentList = new LinkedList(); + for (final ContentItem item : itemList) { + if (item instanceof TextItem textItem) { + contentList.add(TextContent.create().type(TextContent.TypeEnum.TEXT).text(textItem.text())); + } else if (item instanceof ImageItem imageItem) { + final var detail = imageItem.detailLevel().toString(); + final var img = ImageContentImageUrl.create().url(imageItem.imageUrl()).detail(detail); + contentList.add(ImageContent.create().type(ImageContent.TypeEnum.IMAGE_URL).imageUrl(img)); + } + } + return MultiChatMessage.create().role(role()).content(contentList); } /** @@ -66,5 +99,5 @@ default ChatMessage createChatMessage() { */ @Nonnull @Beta - String content(); + MessageContent content(); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContent.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContent.java new file mode 100644 index 000000000..af182385b --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContent.java @@ -0,0 +1,36 @@ +package com.sap.ai.sdk.orchestration; + +import com.sap.ai.sdk.orchestration.model.ImageContent; +import com.sap.ai.sdk.orchestration.model.MultiChatMessageContent; +import com.sap.ai.sdk.orchestration.model.TextContent; +import java.util.List; +import javax.annotation.Nonnull; + +/** + * Represents the content of a chat message. + * + * @param items a list of the content items + * @since 1.3.0 + */ +public record MessageContent(@Nonnull List items) { + @Nonnull + static MessageContent fromMCMContentList( + @Nonnull final List mCMContentList) { + final var itemList = + mCMContentList.stream() + .map( + content -> { + if (content instanceof TextContent text) { + return new TextItem(text.getText()); + } else { + final var imageUrl = ((ImageContent) content).getImageUrl(); + return (ContentItem) + new ImageItem( + imageUrl.getUrl(), + ImageItem.DetailLevel.fromString(imageUrl.getDetail())); + } + }) + .toList(); + return new MessageContent(itemList); + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java index 963f79ad8..d48a73ca7 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java @@ -6,6 +6,7 @@ import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.LLMChoice; import com.sap.ai.sdk.orchestration.model.LLMModuleResultSynchronous; +import com.sap.ai.sdk.orchestration.model.MultiChatMessage; import com.sap.ai.sdk.orchestration.model.SingleChatMessage; import com.sap.ai.sdk.orchestration.model.TokenUsage; import java.util.ArrayList; @@ -51,33 +52,47 @@ public TokenUsage getTokenUsage() { /** * Get all messages. This can be used for subsequent prompts as a message history. * - * @throws UnsupportedOperationException if the MultiChatMessage type message in chat. + * @throws IllegalArgumentException if the MultiChatMessage type message in chat. * @return A list of all messages. */ @Nonnull - public List getAllMessages() throws UnsupportedOperationException { + public List getAllMessages() throws IllegalArgumentException { final var messages = new ArrayList(); - for (final ChatMessage chatMessage : originalResponse.getModuleResults().getTemplating()) { if (chatMessage instanceof SingleChatMessage simpleMsg) { - final var message = - switch (simpleMsg.getRole()) { - case "user" -> new UserMessage(simpleMsg.getContent()); - case "assistant" -> new AssistantMessage(simpleMsg.getContent()); - case "system" -> new SystemMessage(simpleMsg.getContent()); - default -> throw new IllegalStateException("Unexpected role: " + simpleMsg.getRole()); - }; - messages.add(message); + messages.add(chatMessageIntoMessage(simpleMsg)); + } else if (chatMessage instanceof MultiChatMessage mCMessage) { + messages.add(chatMessageIntoMessage(mCMessage)); } else { - throw new UnsupportedOperationException( - "Messages of MultiChatMessage type not supported by convenience API"); + throw new IllegalArgumentException( + "Messages of type " + chatMessage.getClass() + " are not supported by convenience API"); } } - - messages.add(new AssistantMessage(getChoice().getMessage().getContent())); + messages.add(Message.assistant(getChoice().getMessage().getContent())); return messages; } + @Nonnull + private Message chatMessageIntoMessage(@Nonnull final SingleChatMessage simpleMsg) { + return switch (simpleMsg.getRole()) { + case "user" -> Message.user(simpleMsg.getContent()); + case "assistant" -> Message.assistant(simpleMsg.getContent()); + case "system" -> Message.system(simpleMsg.getContent()); + default -> throw new IllegalStateException("Unexpected role: " + simpleMsg.getRole()); + }; + } + + @Nonnull + private Message chatMessageIntoMessage(@Nonnull final MultiChatMessage mCMessage) { + return switch (mCMessage.getRole()) { + case "user" -> new UserMessage(MessageContent.fromMCMContentList(mCMessage.getContent())); + case "system" -> new SystemMessage(MessageContent.fromMCMContentList(mCMessage.getContent())); + default -> + throw new IllegalStateException( + "Unexpected role with complex message: " + mCMessage.getRole()); + }; + } + /** * Get the LLM response. Useful for accessing the finish reason or further data like logprobs. * diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java index 901b212b1..6facc161d 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java @@ -1,8 +1,13 @@ package com.sap.ai.sdk.orchestration; +import com.google.common.annotations.Beta; +import java.util.LinkedList; +import java.util.List; import javax.annotation.Nonnull; +import lombok.Getter; import lombok.Value; import lombok.experimental.Accessors; +import lombok.experimental.Tolerate; /** Represents a chat message as 'system' to the orchestration service. */ @Value @@ -13,5 +18,31 @@ public class SystemMessage implements Message { @Nonnull String role = "system"; /** The content of the message. */ - @Nonnull String content; + @Nonnull + @Getter(onMethod_ = @Beta) + MessageContent content; + + /** + * Creates a new system message from a string. + * + * @param message the first message. + */ + @Tolerate + public SystemMessage(@Nonnull final String message) { + content = new MessageContent(List.of(new TextItem(message))); + } + + /** + * Add text to the message. + * + * @param message the text to add. + * @return the new message. + * @since 1.3.0 + */ + @Nonnull + public SystemMessage withText(@Nonnull final String message) { + final var contentItems = new LinkedList<>(content.items()); + contentItems.add(new TextItem(message)); + return new SystemMessage(new MessageContent(contentItems)); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java new file mode 100644 index 000000000..d8119683a --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java @@ -0,0 +1,11 @@ +package com.sap.ai.sdk.orchestration; + +import javax.annotation.Nonnull; + +/** + * Represents a text item in a {@link MessageContent} object. + * + * @param text the text of the item + * @since 1.3.0 + */ +public record TextItem(@Nonnull String text) implements ContentItem {} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java index 84207dee5..27301f5b3 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java @@ -1,8 +1,13 @@ package com.sap.ai.sdk.orchestration; +import com.google.common.annotations.Beta; +import java.util.LinkedList; +import java.util.List; import javax.annotation.Nonnull; +import lombok.Getter; import lombok.Value; import lombok.experimental.Accessors; +import lombok.experimental.Tolerate; /** Represents a chat message as 'user' to the orchestration service. */ @Value @@ -13,5 +18,61 @@ public class UserMessage implements Message { @Nonnull String role = "user"; /** The content of the message. */ - @Nonnull String content; + @Nonnull + @Getter(onMethod_ = @Beta) + MessageContent content; + + /** + * Creates a new user message from a string. + * + * @param message the first message. + */ + @Tolerate + public UserMessage(@Nonnull final String message) { + this(new MessageContent(List.of(new TextItem(message)))); + } + + /** + * Add text to the message. + * + * @param message the text to add. + * @return the new message. + * @since 1.3.0 + */ + @Nonnull + public UserMessage withText(@Nonnull final String message) { + final var contentItems = new LinkedList<>(content.items()); + contentItems.add(new TextItem(message)); + return new UserMessage(new MessageContent(contentItems)); + } + + /** + * Add an image to the message with the given image URL and detail level. + * + * @param imageUrl the URL of the image. + * @param detailLevel the detail level of the image. + * @return the new message. + * @since 1.3.0 + */ + @Nonnull + public UserMessage withImage( + @Nonnull final String imageUrl, @Nonnull final ImageItem.DetailLevel detailLevel) { + final var contentItems = new LinkedList<>(content.items()); + contentItems.add(new ImageItem(imageUrl, detailLevel)); + return new UserMessage(new MessageContent(contentItems)); + } + + /** + * Add an image to the message with the given image URL. + * + * @param imageUrl the URL of the image. + * @return the new message. + * @since 1.3.0 + */ + @Nonnull + public UserMessage withImage(@Nonnull final String imageUrl) { + final var contentItems = new LinkedList<>(content.items()); + contentItems.add(new ImageItem(imageUrl)); + return new UserMessage(new MessageContent(contentItems)); + } } diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationConvenienceUnitTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationConvenienceUnitTest.java new file mode 100644 index 000000000..b838a4fb3 --- /dev/null +++ b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationConvenienceUnitTest.java @@ -0,0 +1,33 @@ +package com.sap.ai.sdk.orchestration; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.junit.jupiter.api.Test; + +public class OrchestrationConvenienceUnitTest { + + @Test + void testMessageConstructionText() { + var userMessageViaStaticFactory = Message.user("Text 1"); + var userMessageViaConstructor = new UserMessage(("Text 1")); + assertThat(userMessageViaStaticFactory).isEqualTo(userMessageViaConstructor); + + var systemMessageViaStaticFactory = Message.system("Text 1"); + var systemMessageViaConstructor = new SystemMessage("Text 1"); + assertThat(systemMessageViaStaticFactory).isEqualTo(systemMessageViaConstructor); + } + + @Test + void testMessageConstructionImage() { + var userMessageOnlyImageConvenience = Message.user(new ImageItem("url")); + var userMessageOnlyImageBase = + new UserMessage(new MessageContent(List.of(new ImageItem("url")))); + assertThat(userMessageOnlyImageBase).isEqualTo(userMessageOnlyImageConvenience); + + var userMessageWithImage = Message.user("Text 1").withImage("url"); + var userMessageWithImageAndDetail = + Message.user("Text 1").withImage("url", ImageItem.DetailLevel.AUTO); + assertThat(userMessageWithImage).isEqualTo(userMessageWithImageAndDetail); + } +} diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfigTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfigTest.java index 9fc63df62..f81300cb4 100644 --- a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfigTest.java +++ b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfigTest.java @@ -169,7 +169,7 @@ void testGroundingPrompt() { var prompt = Grounding.create().createGroundingPrompt("Hello, World!"); assertThat(prompt.getMessages()).hasSize(1); var message = prompt.getMessages().get(0); - assertThat(message.content()) + assertThat(((TextItem) message.content().items().get(0)).text()) .isEqualTo( "{{?userMessage}} Use the following information as additional context: {{?groundingContext}}"); } diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java index d98292ad7..1095d4121 100644 --- a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java +++ b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java @@ -31,14 +31,9 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; import com.github.tomakehurst.wiremock.stubbing.Scenario; -import com.sap.ai.sdk.orchestration.model.ChatMessage; -import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; -import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.DPIEntities; import com.sap.ai.sdk.orchestration.model.DataRepositoryType; import com.sap.ai.sdk.orchestration.model.DocumentGroundingFilter; @@ -46,21 +41,12 @@ import com.sap.ai.sdk.orchestration.model.GroundingFilterSearchConfiguration; import com.sap.ai.sdk.orchestration.model.GroundingModuleConfig; import com.sap.ai.sdk.orchestration.model.GroundingModuleConfigConfig; -import com.sap.ai.sdk.orchestration.model.ImageContent; -import com.sap.ai.sdk.orchestration.model.ImageContentImageUrl; import com.sap.ai.sdk.orchestration.model.KeyValueListPair; -import com.sap.ai.sdk.orchestration.model.LLMModuleConfig; -import com.sap.ai.sdk.orchestration.model.LLMModuleResult; import com.sap.ai.sdk.orchestration.model.LLMModuleResultSynchronous; import com.sap.ai.sdk.orchestration.model.LlamaGuard38b; -import com.sap.ai.sdk.orchestration.model.ModuleConfigs; -import com.sap.ai.sdk.orchestration.model.MultiChatMessage; -import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.ai.sdk.orchestration.model.SearchDocumentKeyValueListPair; import com.sap.ai.sdk.orchestration.model.SearchSelectOptionEnum; import com.sap.ai.sdk.orchestration.model.SingleChatMessage; -import com.sap.ai.sdk.orchestration.model.Template; -import com.sap.ai.sdk.orchestration.model.TextContent; import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor; import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Cache; import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination; @@ -72,7 +58,6 @@ import java.util.function.Function; import java.util.stream.Stream; import javax.annotation.Nonnull; -import lombok.SneakyThrows; import org.apache.hc.client5.http.classic.HttpClient; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.io.entity.InputStreamEntity; @@ -221,12 +206,14 @@ void testTemplating() throws IOException { assertThat(response.getRequestId()).isEqualTo("26ea36b5-c196-4806-a9a6-a686f0c6ad91"); final var messageList = result.getAllMessages(); - assertThat(messageList.get(0).content()).isEqualTo("You are a multi language translator"); + assertThat(((TextItem) messageList.get(0).content().items().get(0)).text()) + .isEqualTo("You are a multi language translator"); assertThat(messageList.get(0).role()).isEqualTo("system"); - assertThat(messageList.get(1).content()) + assertThat(((TextItem) messageList.get(1).content().items().get(0)).text()) .isEqualTo("Reply with 'Orchestration Service is working!' in German"); assertThat(messageList.get(1).role()).isEqualTo("user"); - assertThat(messageList.get(2).content()).isEqualTo("Orchestration Service funktioniert!"); + assertThat(((TextItem) messageList.get(2).content().items().get(0)).text()) + .isEqualTo("Orchestration Service funktioniert!"); assertThat(messageList.get(2).role()).isEqualTo("assistant"); var llm = (LLMModuleResultSynchronous) response.getModuleResults().getLlm(); @@ -690,132 +677,109 @@ void streamChatCompletionDeltas() throws IOException { } @Test - void testRequestWithMultiChatMessage() throws IOException { - + void testMultiMessage() throws IOException { stubFor( post("/completion") - .willReturn( - aResponse().withStatus(SC_OK).withBodyFile("multiChatMessageResponse.json"))); - - var multiChatMessage = - MultiChatMessage.create() - .role("user") - .content( - List.of( - TextContent.create() - .type(TextContent.TypeEnum.TEXT) - .text("Can you solve this captcha? Please help me prove my humanity!"), - ImageContent.create() - .type(ImageContent.TypeEnum.IMAGE_URL) - .imageUrl( - ImageContentImageUrl.create().url("https://sample.sap.com/image")))); - - var llmWithImageSupportConfig = - LLMModuleConfig.create() - .modelName(GPT_4O_MINI.getName()) - .modelParams(Map.of()) - .modelVersion(GPT_4O_MINI.getVersion()); - - var templatingModuleConfig = Template.create().template(List.of(multiChatMessage)); - - CompletionPostRequest completionPostRequest = - CompletionPostRequest.create() - .orchestrationConfig( - OrchestrationConfig.create() - .moduleConfigurations( - ModuleConfigs.create() - .llmModuleConfig(llmWithImageSupportConfig) - .templatingModuleConfig(templatingModuleConfig))); - - var response = client.executeRequest(completionPostRequest); + .willReturn(aResponse().withStatus(SC_OK).withBodyFile("multiMessageResponse.json"))); - assertThat(response).isNotNull(); - assertThat(response.getRequestId()).isEqualTo("2547cb86-a143-4064-bf40-45461c6a7ed9"); + var llmWithImageSupportConfig = new OrchestrationModuleConfig().withLlmConfig(GPT_4O_MINI); + + var messageWithTwoTexts = + Message.system("Please answer in exactly two sentences.") + .withText("Start the first sentence with the word 'Well'."); + + var messageWithImage = + Message.user("What is in this image?") + .withText("And what is the main color?") + .withImage( + "https://upload.wikimedia.org/wikipedia/commons/thumb/5/59/SAP_2011_logo.svg/440px-SAP_2011_logo.svg.png"); + var prompt = + new OrchestrationPrompt(messageWithImage).messageHistory(List.of(messageWithTwoTexts)); + + var result = client.chatCompletion(prompt, llmWithImageSupportConfig); + var response = result.getOriginalResponse(); + + assertThat(result.getContent()) + .isEqualTo( + "Well, this image features the logo of SAP, a software company, set against a gradient blue background transitioning from light to dark. The main color in the image is blue."); + assertThat(result.getAllMessages()).hasSize(3); + var systemMessage = result.getAllMessages().get(0); + assertThat(systemMessage.role()).isEqualTo("system"); + assertThat(systemMessage.content().items()).hasSize(2); + assertThat(systemMessage.content().items().get(0)).isInstanceOf(TextItem.class); + assertThat(((TextItem) systemMessage.content().items().get(0)).text()) + .isEqualTo("Please answer in exactly two sentences."); + assertThat(systemMessage.content().items().get(1)).isInstanceOf(TextItem.class); + assertThat(((TextItem) systemMessage.content().items().get(1)).text()) + .isEqualTo("Start the first sentence with the word 'Well'."); + var userMessage = result.getAllMessages().get(1); + assertThat(userMessage.role()).isEqualTo("user"); + assertThat(userMessage.content().items()).hasSize(3); + assertThat(userMessage.content().items().get(0)).isInstanceOf(TextItem.class); + assertThat(((TextItem) userMessage.content().items().get(0)).text()) + .isEqualTo("What is in this image?"); + assertThat(userMessage.content().items().get(1)).isInstanceOf(TextItem.class); + assertThat(((TextItem) userMessage.content().items().get(1)).text()) + .isEqualTo("And what is the main color?"); + assertThat(userMessage.content().items().get(2)).isInstanceOf(ImageItem.class); + assertThat(((ImageItem) userMessage.content().items().get(2)).imageUrl()) + .isEqualTo( + "https://upload.wikimedia.org/wikipedia/commons/thumb/5/59/SAP_2011_logo.svg/440px-SAP_2011_logo.svg.png"); + var assistantMessage = result.getAllMessages().get(2); + assertThat(assistantMessage.role()).isEqualTo("assistant"); + assertThat(assistantMessage.content().items()).hasSize(1); + assertThat(assistantMessage.content().items().get(0)).isInstanceOf(TextItem.class); + assertThat(((TextItem) assistantMessage.content().items().get(0)).text()) + .isEqualTo( + "Well, this image features the logo of SAP, a software company, set against a gradient blue background transitioning from light to dark. The main color in the image is blue."); + assertThat(response).isNotNull(); + assertThat(response.getRequestId()).isEqualTo("8d973a0d-c2cf-437b-a765-08d66bf446d8"); assertThat(response.getModuleResults()).isNotNull(); - assertThat(response.getModuleResults().getTemplating()).hasSize(1); - - var multiChatMessageResponse = - (MultiChatMessage) response.getModuleResults().getTemplating().get(0); - assertThat(((TextContent) multiChatMessageResponse.getContent().get(0)).getText()) - .isEqualTo("Can you solve this captcha? Please help me prove my humanity!"); - assertThat(((TextContent) multiChatMessageResponse.getContent().get(0)).getType()) - .isEqualTo(TextContent.TypeEnum.TEXT); - assertThat(((ImageContent) multiChatMessageResponse.getContent().get(1)).getType()) - .isEqualTo(ImageContent.TypeEnum.IMAGE_URL); - assertThat(((ImageContent) multiChatMessageResponse.getContent().get(1)).getImageUrl().getUrl()) - .isEqualTo("https://sample.sap.com/image"); + assertThat(response.getModuleResults().getTemplating()).hasSize(2); var llmResults = (LLMModuleResultSynchronous) response.getModuleResults().getLlm(); assertThat(llmResults).isNotNull(); - assertThat(llmResults.getId()).isEqualTo("chatcmpl-Annjjf8T5LfLh7PRJPbaUlcC48DdE"); + assertThat(llmResults.getId()).isEqualTo("chatcmpl-AyGx4yLYUH79TK81i21BaABoUpf4v"); assertThat(llmResults.getObject()).isEqualTo("chat.completion"); - assertThat(llmResults.getCreated()).isEqualTo(1736432623); + assertThat(llmResults.getCreated()).isEqualTo(1738928206); assertThat(llmResults.getModel()).isEqualTo("gpt-4o-mini-2024-07-18"); - assertThat(llmResults.getSystemFingerprint()).isEqualTo("fp_5154047bf2"); - + assertThat(llmResults.getSystemFingerprint()).isEqualTo("fp_f3927aa00d"); assertThat(llmResults.getChoices()).hasSize(1); assertThat(llmResults.getChoices().get(0).getMessage().getContent()) .isEqualTo( - "Of course! Just let me put on my human glasses... Oh wait, I left them in the matrix"); + "Well, this image features the logo of SAP, a software company, set against a gradient blue background transitioning from light to dark. The main color in the image is blue."); assertThat(llmResults.getChoices().get(0).getFinishReason()).isEqualTo("stop"); assertThat(llmResults.getChoices().get(0).getMessage().getRole()).isEqualTo("assistant"); assertThat(llmResults.getChoices().get(0).getIndex()).isZero(); - - assertThat(llmResults.getUsage().getCompletionTokens()).isEqualTo(31); - assertThat(llmResults.getUsage().getPromptTokens()).isEqualTo(928); - assertThat(llmResults.getUsage().getTotalTokens()).isEqualTo(959); + assertThat(llmResults.getUsage().getCompletionTokens()).isEqualTo(35); + assertThat(llmResults.getUsage().getPromptTokens()).isEqualTo(250); + assertThat(llmResults.getUsage().getTotalTokens()).isEqualTo(285); var orchestrationResult = (LLMModuleResultSynchronous) response.getOrchestrationResult(); assertThat(orchestrationResult).isNotNull(); - assertThat(orchestrationResult.getId()).isEqualTo("chatcmpl-Annjjf8T5LfLh7PRJPbaUlcC48DdE"); + assertThat(orchestrationResult.getId()).isEqualTo("chatcmpl-AyGx4yLYUH79TK81i21BaABoUpf4v"); assertThat(orchestrationResult.getObject()).isEqualTo("chat.completion"); - assertThat(orchestrationResult.getCreated()).isEqualTo(1736432623); + assertThat(orchestrationResult.getCreated()).isEqualTo(1738928206); assertThat(orchestrationResult.getModel()).isEqualTo("gpt-4o-mini-2024-07-18"); - assertThat(orchestrationResult.getSystemFingerprint()).isEqualTo("fp_5154047bf2"); + assertThat(orchestrationResult.getSystemFingerprint()).isEqualTo("fp_f3927aa00d"); assertThat(orchestrationResult.getChoices()).hasSize(1); assertThat(orchestrationResult.getChoices().get(0).getMessage().getContent()) .isEqualTo( - "Of course! Just let me put on my human glasses... Oh wait, I left them in the matrix"); + "Well, this image features the logo of SAP, a software company, set against a gradient blue background transitioning from light to dark. The main color in the image is blue."); assertThat(orchestrationResult.getChoices().get(0).getFinishReason()).isEqualTo("stop"); assertThat(orchestrationResult.getChoices().get(0).getMessage().getRole()) .isEqualTo("assistant"); assertThat(orchestrationResult.getChoices().get(0).getIndex()).isZero(); - assertThat(orchestrationResult.getUsage().getCompletionTokens()).isEqualTo(31); - assertThat(orchestrationResult.getUsage().getPromptTokens()).isEqualTo(928); - assertThat(orchestrationResult.getUsage().getTotalTokens()).isEqualTo(959); + assertThat(orchestrationResult.getUsage().getCompletionTokens()).isEqualTo(35); + assertThat(orchestrationResult.getUsage().getPromptTokens()).isEqualTo(250); + assertThat(orchestrationResult.getUsage().getTotalTokens()).isEqualTo(285); - try (var requestInputStream = fileLoader.apply("multiChatMessageRequest.json")) { + try (var requestInputStream = fileLoader.apply("multiMessageRequest.json")) { final String requestBody = new String(requestInputStream.readAllBytes()); verify( postRequestedFor(urlPathEqualTo("/completion")) .withRequestBody(equalToJson(requestBody))); } } - - @SneakyThrows - @Test - void testOrchestrationChatResponseWithMultiChatMessage() { - var module = new SimpleModule(); - module.setMixInAnnotation(LLMModuleResult.class, JacksonMixins.NoneTypeInfoMixin.class); - module.addDeserializer( - LLMModuleResult.class, - PolymorphicFallbackDeserializer.fromJsonSubTypes(LLMModuleResult.class)); - module.setMixInAnnotation(ChatMessage.class, JacksonMixins.NoneTypeInfoMixin.class); - module.addDeserializer( - ChatMessage.class, PolymorphicFallbackDeserializer.fromJsonSubTypes(ChatMessage.class)); - - var orchestrationChatResponse = - new OrchestrationChatResponse( - new ObjectMapper() - .registerModule(module) - .readValue( - new String( - fileLoader.apply("__files/multiChatMessageResponse.json").readAllBytes()), - CompletionPostResponse.class)); - - assertThatThrownBy(orchestrationChatResponse::getAllMessages) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("Messages of MultiChatMessage type not supported by convenience API"); - } } diff --git a/orchestration/src/test/resources/__files/multiChatMessageResponse.json b/orchestration/src/test/resources/__files/multiChatMessageResponse.json deleted file mode 100644 index ac8b02669..000000000 --- a/orchestration/src/test/resources/__files/multiChatMessageResponse.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "request_id": "2547cb86-a143-4064-bf40-45461c6a7ed9", - "module_results": { - "templating": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Can you solve this captcha? Please help me prove my humanity!" - }, - { - "type": "image_url", - "image_url": { - "url": "https://sample.sap.com/image", - "detail": "auto" - } - } - ] - } - ], - "llm": { - "id": "chatcmpl-Annjjf8T5LfLh7PRJPbaUlcC48DdE", - "object": "chat.completion", - "created": 1736432623, - "model": "gpt-4o-mini-2024-07-18", - "system_fingerprint": "fp_5154047bf2", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Of course! Just let me put on my human glasses... Oh wait, I left them in the matrix" - }, - "finish_reason": "stop" - } - ], - "usage": { - "completion_tokens": 31, - "prompt_tokens": 928, - "total_tokens": 959 - } - } - }, - "orchestration_result": { - "id": "chatcmpl-Annjjf8T5LfLh7PRJPbaUlcC48DdE", - "object": "chat.completion", - "created": 1736432623, - "model": "gpt-4o-mini-2024-07-18", - "system_fingerprint": "fp_5154047bf2", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Of course! Just let me put on my human glasses... Oh wait, I left them in the matrix" - }, - "finish_reason": "stop" - } - ], - "usage": { - "completion_tokens": 31, - "prompt_tokens": 928, - "total_tokens": 959 - } - } -} \ No newline at end of file diff --git a/orchestration/src/test/resources/__files/multiMessageResponse.json b/orchestration/src/test/resources/__files/multiMessageResponse.json new file mode 100644 index 000000000..c04de51ba --- /dev/null +++ b/orchestration/src/test/resources/__files/multiMessageResponse.json @@ -0,0 +1,84 @@ +{ + "request_id": "8d973a0d-c2cf-437b-a765-08d66bf446d8", + "module_results": { + "templating": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Please answer in exactly two sentences." + }, + { + "type": "text", + "text": "Start the first sentence with the word 'Well'." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is in this image?" + }, + { + "type": "text", + "text": "And what is the main color?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/59/SAP_2011_logo.svg/440px-SAP_2011_logo.svg.png", + "detail": "auto" + } + } + ] + } + ], + "llm": { + "id": "chatcmpl-AyGx4yLYUH79TK81i21BaABoUpf4v", + "object": "chat.completion", + "created": 1738928206, + "model": "gpt-4o-mini-2024-07-18", + "system_fingerprint": "fp_f3927aa00d", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Well, this image features the logo of SAP, a software company, set against a gradient blue background transitioning from light to dark. The main color in the image is blue." + }, + "finish_reason": "stop" + } + ], + "usage": { + "completion_tokens": 35, + "prompt_tokens": 250, + "total_tokens": 285 + } + } + }, + "orchestration_result": { + "id": "chatcmpl-AyGx4yLYUH79TK81i21BaABoUpf4v", + "object": "chat.completion", + "created": 1738928206, + "model": "gpt-4o-mini-2024-07-18", + "system_fingerprint": "fp_f3927aa00d", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Well, this image features the logo of SAP, a software company, set against a gradient blue background transitioning from light to dark. The main color in the image is blue." + }, + "finish_reason": "stop" + } + ], + "usage": { + "completion_tokens": 35, + "prompt_tokens": 250, + "total_tokens": 285 + } + } +} \ No newline at end of file diff --git a/orchestration/src/test/resources/multiChatMessageRequest.json b/orchestration/src/test/resources/multiChatMessageRequest.json deleted file mode 100644 index 315ee8cf6..000000000 --- a/orchestration/src/test/resources/multiChatMessageRequest.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "orchestration_config": { - "module_configurations": { - "llm_module_config": { - "model_name": "gpt-4o-mini", - "model_params": {}, - "model_version": "latest" - }, - "templating_module_config": { - "template": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Can you solve this captcha? Please help me prove my humanity!" - }, - { - "type": "image_url", - "image_url": { - "url": "https://sample.sap.com/image", - "detail": "auto" - } - } - ] - } - ], - "defaults" : { }, - "tools" : [ ] - } - }, - "stream": false - }, - "input_params": {} -} \ No newline at end of file diff --git a/orchestration/src/test/resources/multiMessageRequest.json b/orchestration/src/test/resources/multiMessageRequest.json new file mode 100644 index 000000000..c88a4d7f6 --- /dev/null +++ b/orchestration/src/test/resources/multiMessageRequest.json @@ -0,0 +1,44 @@ + +{ + "orchestration_config" : { + "module_configurations" : { + "llm_module_config" : { + "model_name" : "gpt-4o-mini", + "model_params" : { }, + "model_version" : "latest" + }, + "templating_module_config" : { + "template" : [ { + "role" : "user", + "content" : [ { + "type" : "text", + "text" : "What is in this image?" + }, { + "type" : "text", + "text" : "And what is the main color?" + }, { + "type" : "image_url", + "image_url" : { + "url" : "https://upload.wikimedia.org/wikipedia/commons/thumb/5/59/SAP_2011_logo.svg/440px-SAP_2011_logo.svg.png", + "detail" : "auto" + } + } ] + } ], + "defaults" : { }, + "tools" : [ ] + } + }, + "stream" : false + }, + "input_params" : { }, + "messages_history" : [ { + "role" : "system", + "content" : [ { + "type" : "text", + "text" : "Please answer in exactly two sentences." + }, { + "type" : "text", + "text" : "Start the first sentence with the word 'Well'." + } ] + } ] +} \ No newline at end of file diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java index 99a902954..e34cd18bd 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java @@ -8,6 +8,7 @@ import com.sap.ai.sdk.orchestration.OrchestrationClientException; import com.sap.ai.sdk.orchestration.model.DPIEntities; import com.sap.cloud.sdk.cloudplatform.thread.ThreadContextExecutors; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -189,4 +190,28 @@ Object grounding( } return response.getContent(); } + + @GetMapping("/image") + @Nonnull + Object imageInput(@RequestParam(value = "format", required = false) final String format) { + final var response = + service.imageInput( + "https://upload.wikimedia.org/wikipedia/commons/thumb/5/59/SAP_2011_logo.svg/440px-SAP_2011_logo.svg.png"); + if ("json".equals(format)) { + return response; + } + return response.getContent(); + } + + @GetMapping("/multiString") + @Nonnull + Object multiStringInput(@RequestParam(value = "format", required = false) final String format) { + final var response = + service.multiStringInput( + List.of("What is the capital of France?", "What is Chess about?", "What is 2+2?")); + if ("json".equals(format)) { + return response; + } + return response.getContent(); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index d39f49981..3a2d16cac 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -1,6 +1,7 @@ package com.sap.ai.sdk.app.services; import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GEMINI_1_5_FLASH; +import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GPT_4O_MINI; import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.Parameter.TEMPERATURE; import com.sap.ai.sdk.core.AiCoreService; @@ -8,6 +9,7 @@ import com.sap.ai.sdk.orchestration.AzureFilterThreshold; import com.sap.ai.sdk.orchestration.DpiMasking; import com.sap.ai.sdk.orchestration.Grounding; +import com.sap.ai.sdk.orchestration.ImageItem; import com.sap.ai.sdk.orchestration.LlamaGuardFilter; import com.sap.ai.sdk.orchestration.Message; import com.sap.ai.sdk.orchestration.OrchestrationChatResponse; @@ -53,6 +55,35 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) return client.chatCompletion(prompt, config); } + /** + * Chat request to OpenAI through the Orchestration service with an image. + * + * @return the assistant response object + */ + @Nonnull + public OrchestrationChatResponse imageInput(@Nonnull final String pathToImage) { + final var llmWithImageSupportConfig = + new OrchestrationModuleConfig().withLlmConfig(GPT_4O_MINI); + + final var multiMessage = + Message.user("What is in this image?").withImage(pathToImage, ImageItem.DetailLevel.LOW); + final var prompt = new OrchestrationPrompt(multiMessage); + return client.chatCompletion(prompt, llmWithImageSupportConfig); + } + + /** + * Chat request to OpenAI through the Orchestration service with multiple strings. + * + * @return the assistant response object + */ + @Nonnull + public OrchestrationChatResponse multiStringInput(@Nonnull final List questions) { + final var multiMessage = + Message.user(questions.get(0)).withText(questions.get(1)).withText(questions.get(2)); + final var prompt = new OrchestrationPrompt(multiMessage); + return client.chatCompletion(prompt, config); + } + /** * Asynchronous stream of an OpenAI chat request * diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index 73d599eee..c28a04229 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -87,410 +87,486 @@
-
-
- AI SDK Logo -

Java AI SDK Application

-
-
-
-
-
-

🧰 AI Core

- The AI Core API provides tools to manage the lifecycle of your own AI scenarios, including artifacts, pipeline execution, and scalable deployments for training and inference. - For more information, check the AI Core documentation. -
-
-
Java Specific Deployments
-
    -
  • -
    - -
    - Create and delete a deployment with the Java specific configuration ID. +
    +
    + AI SDK Logo +

    Java AI SDK Application

    +
    +
    +
    +
    +
    +

    🧰 AI Core

    + The AI Core API provides tools to manage the lifecycle of your own AI scenarios, including + artifacts, pipeline execution, and scalable deployments for training and inference. + For more information, check the AI Core documentation. +
    +
    +
    Java Specific Deployments
    +
      +
    • +
      + +
      + Create and delete a deployment with the Java specific configuration ID. +
      -
    -
    • -
  • -
    - -
    - Stop all deployments with the Java specific configuration ID. Only RUNNING deployments can be STOPPED. +
    • +
  • +
    + +
    + Stop all deployments with the Java specific configuration ID. Only RUNNING + deployments can be STOPPED. +
    -
- -
  • -
    - -
    - Delete all deployments with the Java specific configuration ID. Only UNKNOWN and STOPPED deployments can be DELETED +
    • +
  • +
    + +
    + Delete all deployments with the Java specific configuration ID. Only UNKNOWN and + STOPPED deployments can be DELETED +
    -
    • - -
  • -
    - -
    - Get all deployments with the Java specific configuration ID. +
    • +
  • +
    + +
    + Get all deployments with the Java specific configuration ID. +
    -
    • - - -
    -
    All Deployments
    -
      -
    • -
      - -
      - Get all deployments, including non-Java specific deployments. +
      • +
    +
    +
    All Deployments
    +
      +
    • +
      + +
      + Get all deployments, including non-Java specific deployments. +
      -
    - - -
    -
    Scenarios
    -
      -
    • -
      - -
      - Get the list of available scenarios. +
      • +
    +
    +
    Scenarios
    +
      +
    • +
      + +
      + Get the list of available scenarios. +
      -
    - -
  • -
    - -
    - Get the list of available models. +
    • +
  • +
    + +
    + Get the list of available models. +
    - -
    • - -
    -
    Configurations
    -
      -
    • -
      - -
      - Get the list of configurations. +
      • +
    +
    +
    Configurations
    +
      +
    • +
      + +
      + Get the list of configurations. +
      - -
      • -
    + + + - -
    -
    -
    -
    - Orchestration Logo -

    Orchestration

    +
    +
    +
    +
    + Orchestration Logo +

    Orchestration

    +
    + The Orchestration API offers functionality for enhancing your LLM calls with Templating, + Filtering, Data Masking, Grounding and more. + For more information, check the Orchestration documentation
    - The Orchestration API offers functionality for enhancing your LLM calls with Templating, Filtering, Data Masking, Grounding and more. - For more information, check the Orchestration documentation -
    -
    -
      -
    • -
      - -
      - Chat request to an LLM through the Orchestration service with a simple prompt. +
      +
        +
      • +
        + +
        + Chat request to an LLM through the Orchestration service with a simple prompt. +
        -
      -
      • -
    • -
      - -
      - Asynchronous stream of an LLM chat request. +
      • +
    • +
      + +
      + Asynchronous stream of an LLM chat request. +
      -
    - -
  • -
    - -
    - Chat request to an LLM through the Orchestration service with a template. +
    • +
  • +
    + +
    + Chat request to an LLM through the Orchestration service with a template. +
    -
    • - -
  • -
    - -
    - Chat request to an LLM through the Orchestration service using message history. +
    • +
  • +
    + +
    + Chat request to an LLM through the Orchestration service using message history. +
    -
    • - + -
  • -
    - -
    - Apply lenient input filtering for a request to orchestration. +
  • +
    + +
    + Apply lenient input filtering for a request to orchestration. +
    +
    +
    • +
  • +
    + +
    + Apply lenient output filtering for a request to orchestration. +
    +
    +
    • +
  • +
    + +
    + Apply lenient input filtering for a request to orchestration. +
    -
    • - -
  • -
    - -
    - Apply lenient output filtering for a request to orchestration. +
    • +
  • +
    + +
    + Let the orchestration service evaluate the feedback on the AI SDK provided by a + hypothetical + user. Anonymize any names given as they are not relevant for judging the + sentiment of the + feedback. +
    -
    • - -
  • -
    - -
    - Apply lenient input filtering for a request to orchestration. +
    • +
  • +
    + +
    + Let the orchestration service a response to a hypothetical user who provided + feedback on the AI + SDK. Pseudonymize the user's name and location to protect their privacy. +
    -
    • - -
  • -
    - -
    - Let the orchestration service evaluate the feedback on the AI SDK provided by a hypothetical - user. Anonymize any names given as they are not relevant for judging the sentiment of the - feedback. +
    • +
  • +
    + +
    + Using grounding to provide additional context to the AI model. +
    - -
    • -
  • -
    - -
    - Let the orchestration service a response to a hypothetical user who provided feedback on the AI - SDK. Pseudonymize the user's name and location to protect their privacy. +
    • + +
  • +
    + +
    + Chat request to an LLM through the Orchestration service with image as input. +
    - -
    • -
  • -
    - -
    - Using grounding to provide additional context to the AI model. +
    • +
  • +
    + +
    + Chat request to an LLM through the Orchestration service with multiple texts. +
    - -
    • - + + + - -
    -
    -
    -

    🧑‍🔬 Foundation Models

    - The Foundation Models API offers functionality for directly calling a foundation model and using the features provided by its own API. - For more information, check the Foundation Models documentation. -
    -
    -
    - OpenAI Logo -
    OpenAI
    +
    +
    +
    +

    🧑‍🔬 Foundation Models

    + The Foundation Models API offers functionality for directly calling a foundation model and using + the features provided by its own API. + For more information, check the Foundation Models documentation.
    -
      -
    • -
      - -
      - Chat request to OpenAI. +
      +
      + OpenAI Logo +
      OpenAI
      +
      +
        +
      • +
        + +
        + Chat request to OpenAI. +
        -
      -
      • -
    • -
      - -
      - Asynchronous stream of an OpenAI chat request. +
      • +
    • +
      + +
      + Asynchronous stream of an OpenAI chat request. +
      -
    - -
  • -
    - -
    - Asynchronous stream of an OpenAI chat request and provision of complete response. +
    • +
  • +
    + +
    + Asynchronous stream of an OpenAI chat request and provision of complete + response. +
    -
    • - -
  • -
    - -
    - Chat request to OpenAI with a tool. +
    • +
  • +
    + +
    + Chat request to OpenAI with a tool. +
    -
    • - -
  • -
    - -
    - Chat request to OpenAI with an image. +
    • +
  • +
    + +
    + Chat request to OpenAI with an image. +
    -
    • - -
  • -
    - -
    - Get the embedding of a text. +
    • +
  • +
    + +
    + Get the embedding of a text. +
    -
    • - -
  • -
    - -
    - Chat request to OpenAI filtering by resource group. +
    • +
  • +
    + +
    + Chat request to OpenAI filtering by resource group. +
    -
    • - - + + + - -
    -
    -
    -
    - Spring Logo -

    Spring AI

    +
    +
    +
    +
    + Spring Logo +

    Spring AI

    +
    + Our clients are integrated within the Spring AI framework
    - Our clients are integrated within the Spring AI framework -
    -
    -
    - Orchestration Logo -
    Orchestration Integration
    -
    -
      -
    • -
      - -
      - Chat request to an LLM through the Orchestration service with a simple prompt. +
      +
      + Orchestration Logo +
      Orchestration Integration
      +
      +
        +
      • +
        + +
        + Chat request to an LLM through the Orchestration service with a simple prompt. +
        -
      -
      • -
    • -
      - -
      - Asynchronous stream of an LLM chat request through the Orchestration service. +
      • +
    • +
      + +
      + Asynchronous stream of an LLM chat request through the Orchestration service. +
      -
    - -
  • -
    - -
    - Chat request to an LLM through the Orchestration service with a Spring AI template. +
    • +
  • +
    + +
    + Chat request to an LLM through the Orchestration service with a Spring AI + template. +
    -
    • - -
  • -
    - -
    - Let the orchestration service evaluate the feedback on the AI SDK provided by a hypothetical - user. Anonymize any names given as they are not relevant for judging the sentiment of the - feedback. +
    • +
  • +
    + +
    + Let the orchestration service evaluate the feedback on the AI SDK provided by a + hypothetical + user. Anonymize any names given as they are not relevant for judging the + sentiment of the + feedback. +
    -
    • - - + + +
    - -
    -
    - - - - -
    -
    - GitHub -
    -
    - ❓ -
    - Welcome to the Java AI SDK Application. In this Demo and End-to-End Test App you can test several - endpoints of our API. For more information, check our GitHub repository linked at the bottom. +
    +
    + + + +
    -
    -
    +
    + GitHub +
    +
    + ❓ +
    + Welcome to the Java AI SDK Application. In this Demo and End-to-End Test App you can test several + endpoints of our API. For more information, check our GitHub repository linked at the bottom. +
    +
    +
    diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java index 2928961f4..9df08eb69 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java @@ -8,10 +8,16 @@ import com.sap.ai.sdk.orchestration.OrchestrationClient; import com.sap.ai.sdk.orchestration.OrchestrationClientException; import com.sap.ai.sdk.orchestration.OrchestrationPrompt; +import com.sap.ai.sdk.orchestration.TextItem; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.DPIEntities; import com.sap.ai.sdk.orchestration.model.LLMChoice; import com.sap.ai.sdk.orchestration.model.LLMModuleResultSynchronous; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; @@ -66,7 +72,7 @@ void testTemplate() { final var response = result.getOriginalResponse(); assertThat(response.getRequestId()).isNotEmpty(); - assertThat(result.getAllMessages().get(0).content()) + assertThat(((TextItem) result.getAllMessages().get(0).content().items().get(0)).text()) .isEqualTo("Reply with 'Orchestration Service is working!' in German"); assertThat(result.getAllMessages().get(0).role()).isEqualTo("user"); var llm = (LLMModuleResultSynchronous) response.getModuleResults().getLlm(); @@ -248,4 +254,45 @@ void testLlamaGuardDisabled() { var filterResult = response.getOriginalResponse().getModuleResults().getInputFiltering(); assertThat(filterResult.getMessage()).contains("passed"); } + + @Test + void testImageInput() { + final var result = + service + .imageInput( + "https://upload.wikimedia.org/wikipedia/commons/thumb/5/59/SAP_2011_logo.svg/440px-SAP_2011_logo.svg.png") + .getOriginalResponse(); + final var choices = ((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices(); + assertThat(choices.get(0).getMessage().getContent()).isNotEmpty(); + } + + @Test + void testImageInputBase64() { + String dataUrl = ""; + try { + URL url = new URL("https://upload.wikimedia.org/wikipedia/commons/c/c9/Sap-logo-700x700.jpg"); + try (InputStream inputStream = url.openStream()) { + byte[] imageBytes = inputStream.readAllBytes(); + byte[] encodedBytes = Base64.getEncoder().encode(imageBytes); + String encodedString = new String(encodedBytes, StandardCharsets.UTF_8); + dataUrl = "data:image/jpeg;base64," + encodedString; + } + } catch (Exception e) { + System.out.println("Error fetching or reading the image from URL: " + e.getMessage()); + } + final var result = service.imageInput(dataUrl).getOriginalResponse(); + final var choices = ((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices(); + assertThat(choices.get(0).getMessage().getContent()).isNotEmpty(); + } + + @Test + void testMultiStringInput() { + final var result = + service + .multiStringInput( + List.of("What is the capital of France?", "What is Chess about?", "What is 2+2?")) + .getOriginalResponse(); + final var choices = ((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices(); + assertThat(choices.get(0).getMessage().getContent()).isNotEmpty(); + } }