feat: Orchestration image support

orchestration/src/main/java/com/sap/ai/sdk/orchestration/AssistantMessage.java

@@ -1,5 +1,7 @@
 package com.sap.ai.sdk.orchestration;
 
+import com.sap.ai.sdk.orchestration.model.MultiChatMessageContent;
+import java.util.List;
 import javax.annotation.Nonnull;
 import lombok.Value;
 import lombok.experimental.Accessors;
@@ -13,5 +15,42 @@ public class AssistantMessage implements Message {
   @Nonnull String role = "assistant";
 
   /** The content of the message. */
-  @Nonnull String content;
+  MessageContent content;
+
+  @Nonnull
+  @Override
+  public String content() {
+    return MessageContent.toString(content);
+  }
+
+  @Override
+  @Nonnull
+  public MessageContent getContent() {
+    return content != null ? content : new MessageContentSingle("");
+  }
+
+  public AssistantMessage(String singleMessage) {
+    content = new MessageContentSingle(singleMessage);
+  }
+
+  public AssistantMessage(MessageContent messageContent) {
+    content = messageContent;
+  }
+
+  public AssistantMessage(List<MultiChatMessageContent> multiChatMessageContentList) {
+    content =
+        new MessageContentMulti(
+            multiChatMessageContentList.toArray(MultiChatMessageContent[]::new));
+  }
+
+  @Nonnull
+  public AssistantMessage addTextMessages(@Nonnull String... messages) {
+    return ((AssistantMessage) Message.addTextMessages(content, role, messages));
+  }
+
+  @Nonnull
+  public AssistantMessage addImage(
+      @Nonnull String imageUrl, MultiMessageImageContent.DetailLevel detailLevel) {
+    return ((AssistantMessage) Message.addImage(content, role, imageUrl, detailLevel));
+  }
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java

@@ -2,7 +2,16 @@
 
 import com.google.common.annotations.Beta;
 import com.sap.ai.sdk.orchestration.model.ChatMessage;
+import com.sap.ai.sdk.orchestration.model.ChatMessagesInner;
+import com.sap.ai.sdk.orchestration.model.ImageContent;
+import com.sap.ai.sdk.orchestration.model.ImageContentImageUrl;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessage;
+import com.sap.ai.sdk.orchestration.model.TextContent;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 /** Interface representing convenience wrappers of chat message to the orchestration service. */
 public sealed interface Message permits UserMessage, AssistantMessage, SystemMessage {
@@ -46,8 +55,96 @@ static SystemMessage system(@Nonnull final String msg) {
    * @return the corresponding {@code ChatMessage} object.
    */
   @Nonnull
-  default ChatMessage createChatMessage() {
-    return ChatMessage.create().role(role()).content(content());
+  default ChatMessagesInner createChatMessage() {
+    if (this.getContent() instanceof MessageContentSingle) {
+      return ChatMessage.create().role(role()).content(content());
+    } else if (this.getContent() instanceof MessageContentMulti mCMulti) {
+      return MultiChatMessage.create()
+          .role(role())
+          .content(
+              mCMulti.multiContentList().stream()
+                  .map(
+                      multiMessageContent -> {
+                        if (multiMessageContent instanceof MultiMessageTextContent mMTContent) {
+                          return TextContent.create()
+                              .type(TextContent.TypeEnum.TEXT)
+                              .text(mMTContent.text());
+                        } else if (multiMessageContent
+                            instanceof MultiMessageImageContent mMIContent) {
+                          return ImageContent.create()
+                              .type(ImageContent.TypeEnum.IMAGE_URL)
+                              .imageUrl(
+                                  ImageContentImageUrl.create()
+                                      .url(mMIContent.imageUrl())
+                                      .detail(mMIContent.detailLevel().toString()));
+                        } else {
+                          throw new IllegalArgumentException(
+                              "Unknown subtype of MultiMessageContent: "
+                                  + multiMessageContent.getClass());
+                        }
+                      })
+                  .toList());
+    } else {
+      throw new IllegalArgumentException("Unknown content type: " + this.getContent().getClass());
+    }
+  }
+
+  static Message addTextMessages(
+      @Nullable MessageContent oldContent, @Nonnull String role, @Nonnull String... messages) {
+    if (oldContent == null && messages.length == 1) {
+      return switch (role) {
+        case "user" -> new UserMessage(messages[0]);
+        case "assistant" -> new AssistantMessage(messages[0]);
+        case "system" -> new SystemMessage(messages[0]);
+        default -> throw new IllegalArgumentException("Unknown role: " + role);
+      };
+    }
+    var messagesAsMultiMessageContents =
+        Stream.of(messages).map(MultiMessageTextContent::new).toList();
+    return switch (role) {
+      case "user" ->
+          new UserMessage(createMultiContent(messagesAsMultiMessageContents, oldContent));
+      case "assistant" ->
+          new AssistantMessage(createMultiContent(messagesAsMultiMessageContents, oldContent));
+      case "system" ->
+          new SystemMessage(createMultiContent(messagesAsMultiMessageContents, oldContent));
+      default -> throw new IllegalArgumentException("Unknown role: " + role);
+    };
+  }
+
+  static Message addImage(
+      @Nullable MessageContent oldContent,
+      @Nonnull String role,
+      @Nonnull String imageUrl,
+      @Nonnull MultiMessageImageContent.DetailLevel detailLevel) {
+    return switch (role) {
+      case "user" ->
+          new UserMessage(
+              createMultiContent(
+                  List.of(new MultiMessageImageContent(imageUrl, detailLevel)), oldContent));
+      case "assistant" ->
+          new AssistantMessage(
+              createMultiContent(
+                  List.of(new MultiMessageImageContent(imageUrl, detailLevel)), oldContent));
+      case "system" ->
+          new SystemMessage(
+              createMultiContent(
+                  List.of(new MultiMessageImageContent(imageUrl, detailLevel)), oldContent));
+      default -> throw new IllegalArgumentException("Unknown role: " + role);
+    };
+  }
+
+  private static MessageContentMulti createMultiContent(
+      @Nonnull List<? extends MultiMessageContent> newContent,
+      @Nullable MessageContent oldContent) {
+    var multiContentList = new ArrayList<MultiMessageContent>();
+    if (oldContent instanceof MessageContentSingle mCSingle) {
+      multiContentList.add(new MultiMessageTextContent(mCSingle.content()));
+    } else if (oldContent != null) {
+      multiContentList.addAll(((MessageContentMulti) oldContent).multiContentList());
+    }
+    multiContentList.addAll(newContent);
+    return new MessageContentMulti(multiContentList);
   }
 
   /**
@@ -66,4 +163,16 @@ default ChatMessage createChatMessage() {
   @Nonnull
   @Beta
   String content();
+
+  @Nonnull
+  @Beta
+  MessageContent getContent();
+
+  @Nonnull
+  @Beta
+  Message addTextMessages(@Nonnull String... messages);
+
+  @Nonnull
+  @Beta
+  Message addImage(@Nonnull String imageUrl, MultiMessageImageContent.DetailLevel detailLevel);
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContent.java

@@ -0,0 +1,29 @@
+package com.sap.ai.sdk.orchestration;
+
+public sealed interface MessageContent permits MessageContentSingle, MessageContentMulti {
+
+  static String toString(MessageContent content) {
+    if (content instanceof MessageContentSingle mCSingle) {
+      return mCSingle.content();
+    } else if (content instanceof MessageContentMulti mCMulti) {
+      var strBuilder = new StringBuilder();
+      mCMulti
+          .multiContentList()
+          .forEach(
+              multiContent -> {
+                if (multiContent instanceof MultiMessageTextContent mMText) {
+                  strBuilder.append(mMText.text()).append("; ");
+                } else if (multiContent instanceof MultiMessageImageContent mMImage) {
+                  strBuilder.append(mMImage.imageUrl());
+                } else {
+                  throw new IllegalArgumentException(
+                      "Unknown subtype of MultiChatMessageContent: " + multiContent.getClass());
+                }
+              });
+      return strBuilder.toString();
+    } else {
+      throw new IllegalArgumentException(
+          "Unknown subtype of MessageContent: " + content.getClass());
+    }
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContentMulti.java

@@ -0,0 +1,36 @@
+package com.sap.ai.sdk.orchestration;
+
+import static com.sap.ai.sdk.orchestration.MultiMessageImageContent.DetailLevel;
+
+import com.sap.ai.sdk.orchestration.model.ImageContent;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessageContent;
+import com.sap.ai.sdk.orchestration.model.TextContent;
+import java.util.List;
+
+public record MessageContentMulti(List<MultiMessageContent> multiContentList)
+    implements MessageContent {
+  public MessageContentMulti {}
+
+  public MessageContentMulti(MultiChatMessageContent... multiChatContents) {
+    this(convertIntoMultiMessageList(multiChatContents));
+  }
+
+  private static List<MultiMessageContent> convertIntoMultiMessageList(
+      MultiChatMessageContent... multiChatContents) {
+    List<MultiMessageContent> multiContentList = new java.util.ArrayList<>(List.of());
+    for (MultiChatMessageContent multiChatContent : multiChatContents) {
+      if (multiChatContent instanceof TextContent textContent) {
+        multiContentList.add(new MultiMessageTextContent(textContent.getText()));
+      } else if (multiChatContent instanceof ImageContent imageContent) {
+        var imageUrl = imageContent.getImageUrl();
+        multiContentList.add(
+            new MultiMessageImageContent(
+                imageUrl.getUrl(), DetailLevel.fromString(imageUrl.getDetail())));
+      } else {
+        throw new IllegalArgumentException(
+            "Unknown subtype of MultiChatMessageContent: " + multiChatContent.getClass());
+      }
+    }
+    return multiContentList;
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MessageContentSingle.java

@@ -0,0 +1,3 @@
+package com.sap.ai.sdk.orchestration;
+
+public record MessageContentSingle(String content) implements MessageContent{}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MultiMessageContent.java

@@ -0,0 +1,6 @@
+package com.sap.ai.sdk.orchestration;
+
+public sealed interface MultiMessageContent
+    permits MultiMessageTextContent, MultiMessageImageContent {
+  public String type();
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MultiMessageImageContent.java

@@ -0,0 +1,24 @@
+package com.sap.ai.sdk.orchestration;
+
+public record MultiMessageImageContent(String imageUrl, DetailLevel detailLevel)
+    implements MultiMessageContent {
+  private static final String type = "image_url";
+
+  public String type() {
+    return type;
+  }
+
+  public enum DetailLevel {
+    low,
+    high,
+    auto;
+
+    public static DetailLevel fromString(String str) {
+      try {
+        return DetailLevel.valueOf(str.toLowerCase());
+      } catch (IllegalArgumentException e) {
+        return DetailLevel.low;
+      }
+    }
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/MultiMessageTextContent.java

@@ -0,0 +1,9 @@
+package com.sap.ai.sdk.orchestration;
+
+public record MultiMessageTextContent(String text) implements MultiMessageContent {
+  private static final String type = "text";
+
+  public String type() {
+    return type;
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java

@@ -7,6 +7,7 @@
 import com.sap.ai.sdk.orchestration.model.CompletionPostResponse;
 import com.sap.ai.sdk.orchestration.model.LLMChoice;
 import com.sap.ai.sdk.orchestration.model.LLMModuleResultSynchronous;
+import com.sap.ai.sdk.orchestration.model.MultiChatMessage;
 import com.sap.ai.sdk.orchestration.model.TokenUsage;
 import java.util.ArrayList;
 import java.util.List;
@@ -51,11 +52,11 @@ public TokenUsage getTokenUsage() {
   /**
    * Get all messages. This can be used for subsequent prompts as a message history.
    *
-   * @throws UnsupportedOperationException if the MultiChatMessage type message in chat.
+   * @throws IllegalArgumentException if the MultiChatMessage type message in chat.
    * @return A list of all messages.
    */
   @Nonnull
-  public List<Message> getAllMessages() throws UnsupportedOperationException {
+  public List<Message> getAllMessages() throws IllegalArgumentException {
     final var messages = new ArrayList<Message>();
 
     for (final ChatMessagesInner chatMessage :
@@ -66,12 +67,15 @@ public List<Message> getAllMessages() throws UnsupportedOperationException {
               case "user" -> new UserMessage(simpleMsg.getContent());
               case "assistant" -> new AssistantMessage(simpleMsg.getContent());
               case "system" -> new SystemMessage(simpleMsg.getContent());
-              default -> throw new IllegalStateException("Unexpected role: " + simpleMsg.getRole());
+              default ->
+                  throw new IllegalArgumentException("Unexpected role: " + simpleMsg.getRole());
             };
         messages.add(message);
+      } else if (chatMessage instanceof MultiChatMessage mCMessage) {
+        messages.add(new UserMessage(mCMessage.getContent()));
       } else {
-        throw new UnsupportedOperationException(
-            "Messages of MultiChatMessage type not supported by convenience API");
+        throw new IllegalArgumentException(
+            "Messages of type " + chatMessage.getClass() + " are not supported by convenience API");
       }
     }
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java

@@ -1,5 +1,7 @@
 package com.sap.ai.sdk.orchestration;
 
+import com.sap.ai.sdk.orchestration.model.MultiChatMessageContent;
+import java.util.List;
 import javax.annotation.Nonnull;
 import lombok.Value;
 import lombok.experimental.Accessors;
@@ -13,5 +15,42 @@ public class SystemMessage implements Message {
   @Nonnull String role = "system";
 
   /** The content of the message. */
-  @Nonnull String content;
+  MessageContent content;
+
+  @Nonnull
+  @Override
+  public String content() {
+    return MessageContent.toString(content);
+  }
+
+  @Override
+  @Nonnull
+  public MessageContent getContent() {
+    return content != null ? content : new MessageContentSingle("");
+  }
+
+  public SystemMessage(String singleMessage) {
+    content = new MessageContentSingle(singleMessage);
+  }
+
+  public SystemMessage(MessageContent messageContent) {
+    content = messageContent;
+  }
+
+  public SystemMessage(List<MultiChatMessageContent> multiChatMessageContentList) {
+    content =
+        new MessageContentMulti(
+            multiChatMessageContentList.toArray(MultiChatMessageContent[]::new));
+  }
+
+  @Nonnull
+  public AssistantMessage addTextMessages(@Nonnull String... messages) {
+    return ((AssistantMessage) Message.addTextMessages(content, role, messages));
+  }
+
+  @Nonnull
+  public AssistantMessage addImage(
+      @Nonnull String imageUrl, MultiMessageImageContent.DetailLevel detailLevel) {
+    return ((AssistantMessage) Message.addImage(content, role, imageUrl, detailLevel));
+  }
 }