feat: Orchestration image support

orchestration/src/main/java/com/sap/ai/sdk/orchestration/AssistantMessage.java

@@ -13,5 +13,15 @@ public class AssistantMessage implements Message {
   @Nonnull String role = "assistant";
 
   /** The content of the message. */
-  @Nonnull String content;
+  @Nonnull MessageContentSingle content;
+
+  @Override
+  @Nonnull
+  public MessageContent content() {
+    return content;
+  }
+
+  public AssistantMessage(String singleMessage) {
+    content = new MessageContentSingle(singleMessage);
+  }
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java

@@ -2,6 +2,11 @@
 
 import com.google.common.annotations.Beta;
 import com.sap.ai.sdk.orchestration.model.ChatMessage;
+import com.sap.ai.sdk.orchestration.model.ChatMessagesInner;
+import com.sap.ai.sdk.orchestration.model.ImageContent;
+import com.sap.ai.sdk.orchestration.model.ImageContentImageUrl;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessage;
+import com.sap.ai.sdk.orchestration.model.TextContent;
 import javax.annotation.Nonnull;
 
 /** Interface representing convenience wrappers of chat message to the orchestration service. */
@@ -18,6 +23,17 @@ static UserMessage user(@Nonnull final String msg) {
     return new UserMessage(msg);
   }
 
+  /**
+   * A convenience method to create a user message.
+   *
+   * @param msg the message content.
+   * @return the user message.
+   */
+  @Nonnull
+  static UserMessage user(@Nonnull final MessageContent msg) {
+    return new UserMessage(msg);
+  }
+
   /**
    * A convenience method to create an assistant message.
    *
@@ -40,14 +56,57 @@ static SystemMessage system(@Nonnull final String msg) {
     return new SystemMessage(msg);
   }
 
+  /**
+   * A convenience method to create a system message.
+   *
+   * @param msg the message content.
+   * @return the system message.
+   */
+  @Nonnull
+  static SystemMessage system(@Nonnull final MessageContent msg) {
+    return new SystemMessage(msg);
+  }
+
   /**
    * Converts the message to a serializable ChatMessage object.
    *
    * @return the corresponding {@code ChatMessage} object.
    */
   @Nonnull
-  default ChatMessage createChatMessage() {
-    return ChatMessage.create().role(role()).content(content());
+  default ChatMessagesInner createChatMessage() {
+    if (this.content() instanceof MessageContentSingle) {
+      return ChatMessage.create()
+          .role(role())
+          .content(((MessageContentSingle) content()).content());
+    } else if (this.content() instanceof MessageContentMulti mCMulti) {
+      return MultiChatMessage.create()
+          .role(role())
+          .content(
+              mCMulti.multiContentList().stream()
+                  .map(
+                      multiMessageContent -> {
+                        if (multiMessageContent instanceof MultiMessageTextContent mMTContent) {
+                          return TextContent.create()
+                              .type(TextContent.TypeEnum.TEXT)
+                              .text(mMTContent.text());
+                        } else if (multiMessageContent
+                            instanceof MultiMessageImageContent mMIContent) {
+                          return ImageContent.create()
+                              .type(ImageContent.TypeEnum.IMAGE_URL)
+                              .imageUrl(
+                                  ImageContentImageUrl.create()
+                                      .url(mMIContent.imageUrl())
+                                      .detail(mMIContent.detailLevel().toString()));
+                        } else {
+                          throw new IllegalArgumentException(
+                              "Unknown subtype of MultiMessageContent: "
+                                  + multiMessageContent.getClass());
+                        }
+                      })
+                  .toList());
+    } else {
+      throw new IllegalArgumentException("Unknown content type: " + this.content().getClass());
+    }
   }
 
   /**
@@ -65,5 +124,5 @@ default ChatMessage createChatMessage() {
    */
   @Nonnull
   @Beta
-  String content();
+  Object content();
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContent.java

@@ -0,0 +1,29 @@
+package com.sap.ai.sdk.orchestration;
+
+public sealed interface MessageContent permits MessageContentSingle, MessageContentMulti {
+
+  static String toString(MessageContent content) {
+    if (content instanceof MessageContentSingle mCSingle) {
+      return mCSingle.content();
+    } else if (content instanceof MessageContentMulti mCMulti) {
+      var strBuilder = new StringBuilder();
+      mCMulti
+          .multiContentList()
+          .forEach(
+              multiContent -> {
+                if (multiContent instanceof MultiMessageTextContent mMText) {
+                  strBuilder.append(mMText.text()).append("; ");
+                } else if (multiContent instanceof MultiMessageImageContent mMImage) {
+                  strBuilder.append(mMImage.imageUrl());
+                } else {
+                  throw new IllegalArgumentException(
+                      "Unknown subtype of MultiChatMessageContent: " + multiContent.getClass());
+                }
+              });
+      return strBuilder.toString();
+    } else {
+      throw new IllegalArgumentException(
+          "Unknown subtype of MessageContent: " + content.getClass());
+    }
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContentMulti.java

@@ -0,0 +1,57 @@
+package com.sap.ai.sdk.orchestration;
+
+import static com.sap.ai.sdk.orchestration.MultiMessageImageContent.DetailLevel;
+
+import com.sap.ai.sdk.orchestration.model.ImageContent;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessageContent;
+import com.sap.ai.sdk.orchestration.model.TextContent;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+public record MessageContentMulti(List<MultiMessageContent> multiContentList)
+    implements MessageContent {
+
+  public MessageContentMulti(MultiChatMessageContent... multiChatContents) {
+    this(convertIntoMultiMessageList(multiChatContents));
+  }
+
+  private static List<MultiMessageContent> convertIntoMultiMessageList(
+      MultiChatMessageContent... multiChatContents) {
+    List<MultiMessageContent> multiContentList = new java.util.ArrayList<>(List.of());
+    for (MultiChatMessageContent multiChatContent : multiChatContents) {
+      if (multiChatContent instanceof TextContent textContent) {
+        multiContentList.add(new MultiMessageTextContent(textContent.getText()));
+      } else if (multiChatContent instanceof ImageContent imageContent) {
+        var imageUrl = imageContent.getImageUrl();
+        multiContentList.add(
+            new MultiMessageImageContent(
+                imageUrl.getUrl(), DetailLevel.fromString(imageUrl.getDetail())));
+      } else {
+        throw new IllegalArgumentException(
+            "Unknown subtype of MultiChatMessageContent: " + multiChatContent.getClass());
+      }
+    }
+    return multiContentList;
+  }
+
+  MessageContentMulti(
+      @Nonnull List<? extends MultiMessageContent> newContent,
+      @Nullable MessageContent oldContent) {
+    this(mergeContent(newContent, oldContent));
+  }
+
+  private static List<MultiMessageContent> mergeContent(
+      @Nonnull List<? extends MultiMessageContent> newContent,
+      @Nullable MessageContent oldContent) {
+    var multiContentList = new ArrayList<MultiMessageContent>();
+    if (oldContent instanceof MessageContentSingle mCSingle) {
+      multiContentList.add(new MultiMessageTextContent(mCSingle.content()));
+    } else if (oldContent != null) {
+      multiContentList.addAll(((MessageContentMulti) oldContent).multiContentList());
+    }
+    multiContentList.addAll(newContent);
+    return multiContentList;
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContentSingle.java

@@ -0,0 +1,3 @@
+package com.sap.ai.sdk.orchestration;
+
+public record MessageContentSingle(String content) implements MessageContent {}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MultiMessageContent.java

@@ -0,0 +1,6 @@
+package com.sap.ai.sdk.orchestration;
+
+public sealed interface MultiMessageContent
+    permits MultiMessageTextContent, MultiMessageImageContent {
+  public String type();
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MultiMessageImageContent.java

@@ -0,0 +1,24 @@
+package com.sap.ai.sdk.orchestration;
+
+public record MultiMessageImageContent(String imageUrl, DetailLevel detailLevel)
+    implements MultiMessageContent {
+  private static final String type = "image_url";
+
+  public String type() {
+    return type;
+  }
+
+  public enum DetailLevel {
+    low,
+    high,
+    auto;
+
+    public static DetailLevel fromString(String str) {
+      try {
+        return DetailLevel.valueOf(str.toLowerCase());
+      } catch (IllegalArgumentException e) {
+        return DetailLevel.low;
+      }
+    }
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MultiMessageTextContent.java

@@ -0,0 +1,9 @@
+package com.sap.ai.sdk.orchestration;
+
+public record MultiMessageTextContent(String text) implements MultiMessageContent {
+  private static final String type = "text";
+
+  public String type() {
+    return type;
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java

@@ -7,6 +7,8 @@
 import com.sap.ai.sdk.orchestration.model.CompletionPostResponse;
 import com.sap.ai.sdk.orchestration.model.LLMChoice;
 import com.sap.ai.sdk.orchestration.model.LLMModuleResultSynchronous;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessage;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessageContent;
 import com.sap.ai.sdk.orchestration.model.TokenUsage;
 import java.util.ArrayList;
 import java.util.List;
@@ -51,31 +53,52 @@ public TokenUsage getTokenUsage() {
   /**
    * Get all messages. This can be used for subsequent prompts as a message history.
    *
-   * @throws UnsupportedOperationException if the MultiChatMessage type message in chat.
+   * @throws IllegalArgumentException if the MultiChatMessage type message in chat.
    * @return A list of all messages.
    */
   @Nonnull
-  public List<Message> getAllMessages() throws UnsupportedOperationException {
+  public List<Message> getAllMessages() throws IllegalArgumentException {
     final var messages = new ArrayList<Message>();
 
     for (final ChatMessagesInner chatMessage :
         originalResponse.getModuleResults().getTemplating()) {
       if (chatMessage instanceof ChatMessage simpleMsg) {
         final var message =
             switch (simpleMsg.getRole()) {
-              case "user" -> new UserMessage(simpleMsg.getContent());
-              case "assistant" -> new AssistantMessage(simpleMsg.getContent());
-              case "system" -> new SystemMessage(simpleMsg.getContent());
-              default -> throw new IllegalStateException("Unexpected role: " + simpleMsg.getRole());
+              case "user" -> Message.user(simpleMsg.getContent());
+              case "assistant" -> Message.assistant(simpleMsg.getContent());
+              case "system" -> Message.system(simpleMsg.getContent());
+              default ->
+                  throw new IllegalArgumentException(
+                      "Unexpected role: "
+                          + simpleMsg.getRole()
+                          + ". Only 'user', 'assistant' and 'system' are supported.");
             };
         messages.add(message);
+      } else if (chatMessage instanceof MultiChatMessage mCMessage) {
+        messages.add(
+            switch (mCMessage.getRole()) {
+              case "user" ->
+                  Message.user(
+                      new MessageContentMulti(
+                          mCMessage.getContent().toArray(MultiChatMessageContent[]::new)));
+              case "system" ->
+                  Message.system(
+                      new MessageContentMulti(
+                          mCMessage.getContent().toArray(MultiChatMessageContent[]::new)));
+              default ->
+                  throw new IllegalArgumentException(
+                      "Unexpected role using MultiChatMessage: "
+                          + mCMessage.getRole()
+                          + ". Only 'user' and 'system' are supported.");
+            });
       } else {
-        throw new UnsupportedOperationException(
-            "Messages of MultiChatMessage type not supported by convenience API");
+        throw new IllegalArgumentException(
+            "Messages of type " + chatMessage.getClass() + " are not supported by convenience API");
       }
     }
 
-    messages.add(new AssistantMessage(getChoice().getMessage().getContent()));
+    messages.add(Message.assistant(getChoice().getMessage().getContent()));
     return messages;
   }
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java

@@ -1,5 +1,6 @@
 package com.sap.ai.sdk.orchestration;
 
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.Value;
 import lombok.experimental.Accessors;
@@ -13,5 +14,37 @@ public class SystemMessage implements Message {
   @Nonnull String role = "system";
 
   /** The content of the message. */
-  @Nonnull String content;
+  @Nonnull MessageContent content;
+
+  @Override
+  @Nonnull
+  public MessageContent content() {
+    return content;
+  }
+
+  public SystemMessage(String singleMessage) {
+    content = new MessageContentSingle(singleMessage);
+  }
+
+  SystemMessage(MessageContent messageContent) {
+    if (!(messageContent instanceof MessageContentSingle)) {
+      ((MessageContentMulti) messageContent)
+          .multiContentList().stream()
+              .filter(mCMC -> !(mCMC instanceof MultiMessageTextContent))
+              .findAny()
+              .ifPresent(
+                  mCMC -> {
+                    throw new IllegalArgumentException(
+                        "Only TextContent is supported for SystemMessage");
+                  });
+    }
+    content = messageContent;
+  }
+
+  @Nonnull
+  public SystemMessage addText(@Nonnull String... messages) {
+    return new SystemMessage(
+        new MessageContentMulti(
+            Stream.of(messages).map(MultiMessageTextContent::new).toList(), content));
+  }
 }