feat: Orchestration image support

docs/guides/ORCHESTRATION_CHAT_COMPLETION.md

@@ -259,6 +259,51 @@ try (Stream<String> stream = client.streamChatCompletion(prompt, config)) {
 Please find [an example in our Spring Boot application](../../sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java).
 It shows the usage of Spring Boot's `ResponseBodyEmitter` to stream the chat completion delta messages to the frontend in real-time.
 
+
+## Add images and multiple text inputs to a message
+
+It's possible to add images and multiple text inputs to a message.
+
+### Add images to a message
+
+An image can be added to an existing message as follows.
+
+```java
+var newMessage = message.andImage("https://url.to/image.jpg");
+```
+
+You can also construct a message with an image directly, using the `MessageContent` class.
+
+```java
+var message = Message.user(MessageContent.image("https://url.to/image.jpg"));
+```
+
+Some AI models, like GPT 4o, support additionally setting the detail level with which the image is read. This can be set via the `DetailLevel` parameter.
+
+```java
+var newMessage = message.andImage("https://url.to/image.jpg", ImageItem.DetailLevel.low);
+```
+Note, that currently only user messages are supported for image attachments.
+
+### Add multiple text inputs to a message
+
+It's also possible to add multiple text inputs to a message. This can be useful for providing additional context to the AI model. You can construct a message with multiple text inputs as follows.
+
+```java
+var message = Message.user("What is chess about?", "Answer in two sentences.");
+```
+
+Or you can add additional text inputs to an existing message.
+
+```java
+var newMessage = message.andText("Please use many emojis in your answer.");
+```
+
+Note, that only user and system messages are supported for multiple text inputs.
+
+Please find [an example in our Spring Boot application](../../sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java).
+
+
 ## Set model parameters
 
 Change your LLM configuration to add model parameters:

docs/release-notes/release_notes.md

@@ -8,11 +8,12 @@
 
 ### 🔧 Compatibility Notes
 
--
+- `Message.content()` returns a `ContentItem` now instead of a `String`. Use `((TextItem) Message.content().contentItemList().get(0)).text()` if the corresponding `ContentItem` is a `TextItem` and the string representation is needed.
 
 ### ✨ New Functionality
 
 - [Add Orchestration `LlamaGuardFilter`](../guides/ORCHESTRATION_CHAT_COMPLETION.md#chat-completion-filter).
+- [Convenient methods to create messages containing images and multiple text inputs](../guides/ORCHESTRATION_CHAT_COMPLETION.md#add-images-and-multiple-text-inputs-to-a-message)
 
 ### 📈 Improvements
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/AssistantMessage.java

@@ -1,6 +1,8 @@
 package com.sap.ai.sdk.orchestration;
 
+import com.google.common.annotations.Beta;
 import javax.annotation.Nonnull;
+import lombok.Getter;
 import lombok.Value;
 import lombok.experimental.Accessors;
 
@@ -13,5 +15,16 @@ public class AssistantMessage implements Message {
   @Nonnull String role = "assistant";
 
   /** The content of the message. */
-  @Nonnull String content;
+  @Nonnull
+  @Getter(onMethod_ = @Beta)
+  MessageContent content;
+
+  /**
+   * Creates a new assistant message with the given single message.
+   *
+   * @param singleMessage the single message.
+   */
+  public AssistantMessage(@Nonnull final String singleMessage) {
+    content = new MessageContent(singleMessage);
+  }
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/ContentItem.java

@@ -0,0 +1,4 @@
+package com.sap.ai.sdk.orchestration;
+
+/** Represents an item in a {@link MessageContent} object. */
+public sealed interface ContentItem permits TextItem, ImageItem {}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/ImageItem.java

@@ -0,0 +1,56 @@
+package com.sap.ai.sdk.orchestration;
+
+import java.util.Locale;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * Represents an image item in a {@link MessageContent} object.
+ *
+ * @param imageUrl the URL of the image
+ * @param detailLevel the detail level of the image (optional
+ */
+public record ImageItem(@Nonnull String imageUrl, @Nonnull DetailLevel detailLevel)
+    implements ContentItem {
+
+  /**
+   * Creates a new image item with the given image URL and detail level.
+   *
+   * @param imageUrl the URL of the image
+   * @param detailLevel the detail level of the image
+   */
+  public ImageItem(@Nonnull final String imageUrl, @Nullable final DetailLevel detailLevel) {
+    this.imageUrl = imageUrl;
+    this.detailLevel = detailLevel != null ? detailLevel : DetailLevel.auto;
+  }
+
+  /**
+   * Creates a new image item with the given image URL.
+   *
+   * @param imageUrl the URL of the image
+   */
+  public ImageItem(@Nonnull final String imageUrl) {
+    this(imageUrl, DetailLevel.auto);
+  }
+
+  /** The detail level of the image. */
+  public enum DetailLevel {
+    /** Low detail level. */
+    low,
+    /** High detail level. */
+    high,
+    /** Automatic detail level. */
+    auto;
+
+    /**
+     * Converts a string to a detail level.
+     *
+     * @param str the string to convert
+     * @return the detail level
+     */
+    @Nonnull
+    static DetailLevel fromString(@Nonnull final String str) {
+      return DetailLevel.valueOf(str.toLowerCase(Locale.ENGLISH));
+    }
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java

@@ -2,43 +2,74 @@
 
 import com.google.common.annotations.Beta;
 import com.sap.ai.sdk.orchestration.model.ChatMessage;
+import com.sap.ai.sdk.orchestration.model.ImageContent;
+import com.sap.ai.sdk.orchestration.model.ImageContentImageUrl;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessage;
 import com.sap.ai.sdk.orchestration.model.SingleChatMessage;
+import com.sap.ai.sdk.orchestration.model.TextContent;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 /** Interface representing convenience wrappers of chat message to the orchestration service. */
 public sealed interface Message permits UserMessage, AssistantMessage, SystemMessage {
 
+  /**
+   * A convenience method to create a user message from one or more strings.
+   *
+   * @param message the message content.
+   * @param additionalMessages the additional messages.
+   * @return the user message.
+   */
+  @Nonnull
+  static UserMessage user(
+      @Nonnull final String message, @Nullable final String... additionalMessages) {
+    return new UserMessage(message, additionalMessages);
+  }
+
   /**
    * A convenience method to create a user message.
    *
-   * @param msg the message content.
+   * @param message the message content.
    * @return the user message.
    */
   @Nonnull
-  static UserMessage user(@Nonnull final String msg) {
-    return new UserMessage(msg);
+  static UserMessage user(@Nonnull final MessageContent message) {
+    return new UserMessage(message);
   }
 
   /**
    * A convenience method to create an assistant message.
    *
-   * @param msg the message content.
+   * @param message the message content.
    * @return the assistant message.
    */
   @Nonnull
-  static AssistantMessage assistant(@Nonnull final String msg) {
-    return new AssistantMessage(msg);
+  static AssistantMessage assistant(@Nonnull final String message) {
+    return new AssistantMessage(message);
+  }
+
+  /**
+   * A convenience method to create a system message from one or more strings.
+   *
+   * @param message the message content.
+   * @return the system message.
+   */
+  @Nonnull
+  static SystemMessage system(
+      @Nonnull final String message, @Nullable final String... additionalMessages) {
+    return new SystemMessage(message, additionalMessages);
   }
 
   /**
-   * A convenience method to create a system message.
+   * A convenience method to create a system message. As of now, only text content is supported for
+   * system messages by most AIs.
    *
-   * @param msg the message content.
+   * @param message the message content.
    * @return the system message.
    */
   @Nonnull
-  static SystemMessage system(@Nonnull final String msg) {
-    return new SystemMessage(msg);
+  static SystemMessage system(@Nonnull final MessageContent message) {
+    return new SystemMessage(message);
   }
 
   /**
@@ -48,7 +79,31 @@ static SystemMessage system(@Nonnull final String msg) {
    */
   @Nonnull
   default ChatMessage createChatMessage() {
-    return SingleChatMessage.create().role(role()).content(content());
+    final var itemList = this.content().contentItemList();
+    if (itemList.size() == 1 && itemList.get(0) instanceof TextItem textItem) {
+      return SingleChatMessage.create().role(role()).content(textItem.text());
+    } else {
+      return MultiChatMessage.create()
+          .role(role())
+          .content(
+              itemList.stream()
+                  .map(
+                      contentItem -> {
+                        if (contentItem instanceof ImageItem imageItem) {
+                          return ImageContent.create()
+                              .type(ImageContent.TypeEnum.IMAGE_URL)
+                              .imageUrl(
+                                  ImageContentImageUrl.create()
+                                      .url(imageItem.imageUrl())
+                                      .detail(imageItem.detailLevel().toString()));
+                        } else {
+                          return TextContent.create()
+                              .type(TextContent.TypeEnum.TEXT)
+                              .text(((TextItem) contentItem).text());
+                        }
+                      })
+                  .toList());
+    }
   }
 
   /**
@@ -66,5 +121,5 @@ default ChatMessage createChatMessage() {
    */
   @Nonnull
   @Beta
-  String content();
+  MessageContent content();
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContent.java

@@ -0,0 +1,91 @@
+package com.sap.ai.sdk.orchestration;
+
+import com.sap.ai.sdk.orchestration.model.ImageContent;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessageContent;
+import com.sap.ai.sdk.orchestration.model.TextContent;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * Represents the content of a chat message.
+ *
+ * @param contentItemList a list of the content items
+ */
+public record MessageContent(@Nonnull List<ContentItem> contentItemList) {
+
+  MessageContent(@Nonnull final String singleMessage) {
+    this(List.of(new TextItem(singleMessage)));
+  }
+
+  /**
+   * Creates a new message content containing one {@link ImageItem} with the given image URL and
+   * detail level.
+   *
+   * @param imageUrl the URL of the image
+   * @param detailLevel the detail level of the image
+   * @return the new message content
+   */
+  @Nonnull
+  public static MessageContent image(
+      @Nonnull final String imageUrl, @Nullable final ImageItem.DetailLevel detailLevel) {
+    return new MessageContent(List.of(new ImageItem(imageUrl, detailLevel)));
+  }
+
+  /**
+   * Creates a new message content containing one {@link ImageItem} with the given image URL.
+   *
+   * @param imageUrl the URL of the image
+   * @return the new message content
+   */
+  @Nonnull
+  public static MessageContent image(@Nonnull final String imageUrl) {
+    return new MessageContent(List.of(new ImageItem(imageUrl)));
+  }
+
+  /**
+   * Creates a new message content containing one or more {@link TextItem}s with the given texts.
+   *
+   * @param firstInput the first text of the message
+   * @param moreInputs additional texts of the message
+   * @return the new message content
+   */
+  @Nonnull
+  public static MessageContent text(
+      @Nonnull final String firstInput, @Nullable final String... moreInputs) {
+    return new MessageContent(
+        Stream.concat(
+                Stream.of(firstInput),
+                moreInputs == null ? Stream.empty() : Arrays.stream(moreInputs))
+            .map(text -> (ContentItem) new TextItem(text))
+            .toList());
+  }
+
+  /**
+   * Creates a new message content containing multiple {@link ContentItem}s.
+   *
+   * @param multiChatContents the content items
+   */
+  public MessageContent(@Nonnull final MultiChatMessageContent... multiChatContents) {
+    this(convertIntoMultiMessageList(multiChatContents));
+  }
+
+  @Nonnull
+  private static List<ContentItem> convertIntoMultiMessageList(
+      @Nonnull final MultiChatMessageContent... multiChatContents) {
+    final Function<MultiChatMessageContent, ContentItem> convertMultiChatContent =
+        content -> {
+          if (content instanceof TextContent text) {
+            return new TextItem(text.getText());
+          } else {
+            final var imageUrl = ((ImageContent) content).getImageUrl();
+            return new ImageItem(
+                imageUrl.getUrl(), ImageItem.DetailLevel.fromString(imageUrl.getDetail()));
+          }
+        };
+    return Arrays.stream(multiChatContents).map(convertMultiChatContent).toList();
+  }
+}