feat: [Orchestration] Spec update 12a (#267)

* updated orchestration.yaml

* regenerated model after spec update

* Minimal integration of new features

- convenience api does not support MultiChatMessage

* Fix deserialization with custom addition

- try-catch based approach on Subtype annotation mentions
- Also pass custom list of candidates

* Fix deserialization with custom addition

- try-catch based approach on Subtype annotation mentions
- Also pass custom list of candidates

* Fix codestyle

* release notes for spec update

* Update release_notes.md

remove version naming

* Follow-up update on release_notes.md

* update of release_notes.md implementing review feedback

* Update docs/release-notes/release_notes.md

Co-authored-by: Alexander Dümont <[email protected]>

* Orchestration spec update 12a - Tests (#269)

* Test

- improve coverage on existing tests in addition
- tests for failure cases of PolymorphicFallbackDeserializer

* Unit test request with image (low level api)

* Test calling multi chat response using convenience

* minor test method name change

* Replace url with a sample

---------

Co-authored-by: Roshin Rajan Panackal <[email protected]>

* Improve PolymorphicFallbackDeserializer
- Generalize test case
- add @SInCE and @beta annotation on custom deserializer (public)

* lombok on PolymorphicFallbackDeserializer

* make PolymorphicFallbackDeserializer AllArgsConstructor protected and use chaining

* Add Beta annotation, modify access on deserializer, improve test

- make PolymorphicFallbackDeserializer AllArgsConstructor protected and use chaining
- Add beta annotation to MultiChatMessageContent and ChatMessagesInner
- move request body constraint to stub

* Revert @beta annotation (manual) addition on interfaces

* Revert stub change

---------

Co-authored-by: I750911 <[email protected]>
Co-authored-by: Roshin Rajan Panackal <[email protected]>
Co-authored-by: Alexander Dümont <[email protected]>
Co-authored-by: Roshin Rajan Panackal <[email protected]>

Author: 4 people

docs/release-notes/release_notes.md

@@ -8,15 +8,19 @@
 
 ### 🔧 Compatibility Notes
 
+- `ChatMessage`, as well as new `MultiChatMessage`, are now subtypes of new interface `ChatMessagesInner`.
+  Most variables or methods previously typed as `ChatMessage` in `model` package are now typed as `ChatMessagesInner`. 
 - Add missing `@Beta` annotations to all `com.sap.ai.sdk.core.client` and `com.sap.ai.sdk.core.model` classes.
 
 ### ✨ New Functionality
 
--
+- Orchestration supports images as input in newly introduced `MultiChatMessage`.
+- `MultiChatMessage` also allows for multiple content items (text or image) in one object.
+- Grounding input can be masked with `DPIConfig`.
 
 ### 📈 Improvements
 
--
+- Update Orchestration client to version 0.43.0 (2412a)
 
 ### 🐛 Fixed Issues
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java

@@ -1,6 +1,6 @@
 package com.sap.ai.sdk.orchestration;
 
-import com.sap.ai.sdk.orchestration.model.ChatMessage;
+import com.sap.ai.sdk.orchestration.model.ChatMessagesInner;
 import com.sap.ai.sdk.orchestration.model.CompletionPostRequest;
 import com.sap.ai.sdk.orchestration.model.ModuleConfigs;
 import com.sap.ai.sdk.orchestration.model.OrchestrationConfig;
@@ -32,7 +32,10 @@ static CompletionPostRequest toCompletionPostRequest(
             OrchestrationConfig.create().moduleConfigurations(toModuleConfigs(configCopy)))
         .inputParams(prompt.getTemplateParameters())
         .messagesHistory(
-            prompt.getMessagesHistory().stream().map(Message::createChatMessage).toList());
+            prompt.getMessagesHistory().stream()
+                .map(Message::createChatMessage)
+                .map(ChatMessagesInner.class::cast)
+                .toList());
   }
 
   @Nonnull
@@ -45,7 +48,7 @@ static TemplatingModuleConfig toTemplateModuleConfig(
      * In this case, the request will fail, since the templating module will try to resolve the parameter.
      * To be fixed with https://github.tools.sap/AI/llm-orchestration/issues/662
      */
-    val messages = template instanceof Template t ? t.getTemplate() : List.<ChatMessage>of();
+    val messages = template instanceof Template t ? t.getTemplate() : List.<ChatMessagesInner>of();
     val messagesWithPrompt = new ArrayList<>(messages);
     messagesWithPrompt.addAll(
         prompt.getMessages().stream().map(Message::createChatMessage).toList());

orchestration/src/main/java/com/sap/ai/sdk/orchestration/JacksonMixins.java

@@ -18,4 +18,7 @@ interface LLMModuleResultMixIn {}
   @JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
   @JsonDeserialize(as = LLMChoice.class)
   interface ModuleResultsOutputUnmaskingInnerMixIn {}
+
+  @JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
+  interface NoneTypeInfoMixin {}
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java

@@ -3,6 +3,7 @@
 import static lombok.AccessLevel.PACKAGE;
 
 import com.sap.ai.sdk.orchestration.model.ChatMessage;
+import com.sap.ai.sdk.orchestration.model.ChatMessagesInner;
 import com.sap.ai.sdk.orchestration.model.CompletionPostResponse;
 import com.sap.ai.sdk.orchestration.model.LLMChoice;
 import com.sap.ai.sdk.orchestration.model.LLMModuleResultSynchronous;
@@ -50,21 +51,28 @@ public TokenUsage getTokenUsage() {
   /**
    * Get all messages. This can be used for subsequent prompts as a message history.
    *
+   * @throws UnsupportedOperationException if the MultiChatMessage type message in chat.
    * @return A list of all messages.
    */
   @Nonnull
-  public List<Message> getAllMessages() {
+  public List<Message> getAllMessages() throws UnsupportedOperationException {
     final var messages = new ArrayList<Message>();
 
-    for (final ChatMessage chatMessage : originalResponse.getModuleResults().getTemplating()) {
-      final var message =
-          switch (chatMessage.getRole()) {
-            case "user" -> new UserMessage(chatMessage.getContent());
-            case "assistant" -> new AssistantMessage(chatMessage.getContent());
-            case "system" -> new SystemMessage(chatMessage.getContent());
-            default -> throw new IllegalStateException("Unexpected role: " + chatMessage.getRole());
-          };
-      messages.add(message);
+    for (final ChatMessagesInner chatMessage :
+        originalResponse.getModuleResults().getTemplating()) {
+      if (chatMessage instanceof ChatMessage simpleMsg) {
+        final var message =
+            switch (simpleMsg.getRole()) {
+              case "user" -> new UserMessage(simpleMsg.getContent());
+              case "assistant" -> new AssistantMessage(simpleMsg.getContent());
+              case "system" -> new SystemMessage(simpleMsg.getContent());
+              default -> throw new IllegalStateException("Unexpected role: " + simpleMsg.getRole());
+            };
+        messages.add(message);
+      } else {
+        throw new UnsupportedOperationException(
+            "Messages of MultiChatMessage type not supported by convenience API");
+      }
     }
 
     messages.add(new AssistantMessage(getChoice().getMessage().getContent()));

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java

@@ -5,13 +5,15 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.annotations.Beta;
 import com.sap.ai.sdk.core.AiCoreService;
 import com.sap.ai.sdk.core.DeploymentResolutionException;
 import com.sap.ai.sdk.core.common.ClientResponseHandler;
 import com.sap.ai.sdk.core.common.ClientStreamingHandler;
 import com.sap.ai.sdk.core.common.StreamedDelta;
+import com.sap.ai.sdk.orchestration.model.ChatMessagesInner;
 import com.sap.ai.sdk.orchestration.model.CompletionPostRequest;
 import com.sap.ai.sdk.orchestration.model.CompletionPostResponse;
 import com.sap.ai.sdk.orchestration.model.LLMModuleResult;
@@ -49,6 +51,14 @@ public class OrchestrationClient {
     JACKSON.addMixIn(
         ModuleResultsOutputUnmaskingInner.class,
         JacksonMixins.ModuleResultsOutputUnmaskingInnerMixIn.class);
+
+    final var module =
+        new SimpleModule()
+            .addDeserializer(
+                ChatMessagesInner.class,
+                PolymorphicFallbackDeserializer.fromJsonSubTypes(ChatMessagesInner.class))
+            .setMixInAnnotation(ChatMessagesInner.class, JacksonMixins.NoneTypeInfoMixin.class);
+    JACKSON.registerModule(module);
   }
 
   @Nonnull private final Supplier<HttpDestination> destinationSupplier;

orchestration/src/main/java/com/sap/ai/sdk/orchestration/PolymorphicFallbackDeserializer.java

@@ -0,0 +1,106 @@
+package com.sap.ai.sdk.orchestration;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.google.common.annotations.Beta;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nonnull;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+
+/**
+ * Handles polymorphic deserialization for a base class or interface.
+ *
+ * <p>This deserializer uses a fallback strategy by attempting to deserialize the JSON into known
+ * subtypes. Subtypes are either discovered using the {@link JsonSubTypes} annotation or provided
+ * explicitly. If deserialization fails for all candidates, a {@link JsonMappingException} is thrown
+ * with suppressed exceptions.
+ *
+ * @since 1.2.0
+ * @param <T> The base type for deserialization.
+ */
+@Beta
+@AllArgsConstructor(access = AccessLevel.PROTECTED)
+public class PolymorphicFallbackDeserializer<T> extends JsonDeserializer<T> {
+
+  @Nonnull private final Class<T> baseClass;
+  @Nonnull private final List<Class<? extends T>> candidates;
+
+  /**
+   * Constructs the deserializer using candidates inferred from the {@link JsonSubTypes} annotation.
+   *
+   * @param baseClass The base class or interface to be resolved.
+   * @throws IllegalStateException If no subtypes are found.
+   */
+  @Nonnull
+  protected static <T> PolymorphicFallbackDeserializer<T> fromJsonSubTypes(
+      @Nonnull final Class<T> baseClass) {
+    final var subTypes = baseClass.getAnnotation(JsonSubTypes.class);
+    if (subTypes == null || subTypes.value().length == 0) {
+      throw new IllegalStateException("No subtypes found for " + baseClass.getName());
+    }
+
+    final var candidates = new ArrayList<Class<? extends T>>();
+    for (final var subType : subTypes.value()) {
+      candidates.add((Class<? extends T>) subType.value());
+    }
+
+    return PolymorphicFallbackDeserializer.fromCandidates(baseClass, candidates);
+  }
+
+  /**
+   * Constructs the deserializer with an explicit given list of candidate types.
+   *
+   * @param baseClass The base class or interface to be resolved.
+   * @param candidates A list of candidate classes to try deserialization.
+   */
+  @Nonnull
+  protected static <T> PolymorphicFallbackDeserializer<T> fromCandidates(
+      @Nonnull final Class<T> baseClass, @Nonnull final List<Class<? extends T>> candidates) {
+    return new PolymorphicFallbackDeserializer<>(baseClass, candidates);
+  }
+
+  /**
+   * Deserializes the JSON into the first matching candidate type.
+   *
+   * @param jsonParser The parser providing the JSON.
+   * @param deserializationContext The deserialization context.
+   * @return The deserialized object of a matching candidate type.
+   * @throws JsonMappingException If deserialization fails for all candidates.
+   * @throws IOException If json content cannot be consumed.
+   */
+  @Nonnull
+  @Override
+  public T deserialize(
+      @Nonnull final JsonParser jsonParser,
+      @Nonnull final DeserializationContext deserializationContext)
+      throws IOException {
+
+    final var root = jsonParser.readValueAsTree();
+    final var suppressed = new ArrayList<JsonMappingException>();
+
+    for (final var candidate : candidates) {
+      try {
+        return jsonParser.getCodec().treeToValue(root, candidate);
+      } catch (JsonMappingException e) {
+        suppressed.add(e);
+      }
+    }
+
+    final var aggregateException =
+        JsonMappingException.from(
+            jsonParser,
+            "PolymorphicFallbackDeserializer failed to deserialize "
+                + this.baseClass.getName()
+                + ". Attempted candidates: "
+                + candidates.stream().map(Class::getName).toList());
+
+    suppressed.forEach(aggregateException::addSuppressed);
+    throw aggregateException;
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/model/AzureContentSafety.java

@@ -2,7 +2,7 @@
  * Orchestration
  * Orchestration is an inference service which provides common additional capabilities for business AI scenarios, such as content filtering and data masking. At the core of the service is the LLM module which allows for an easy, harmonized access to the language models of gen AI hub. The service is designed to be modular and extensible, allowing for the addition of new modules in the future. Each module can be configured independently and at runtime, allowing for a high degree of flexibility in the orchestration of AI services.
  *
- * The version of the OpenAPI document: 0.36.1
+ * The version of the OpenAPI document: 0.43.0
  *
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

orchestration/src/main/java/com/sap/ai/sdk/orchestration/model/AzureContentSafetyFilterConfig.java

@@ -2,7 +2,7 @@
  * Orchestration
  * Orchestration is an inference service which provides common additional capabilities for business AI scenarios, such as content filtering and data masking. At the core of the service is the LLM module which allows for an easy, harmonized access to the language models of gen AI hub. The service is designed to be modular and extensible, allowing for the addition of new modules in the future. Each module can be configured independently and at runtime, allowing for a high degree of flexibility in the orchestration of AI services.
  *
- * The version of the OpenAPI document: 0.36.1
+ * The version of the OpenAPI document: 0.43.0
  *
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

orchestration/src/main/java/com/sap/ai/sdk/orchestration/model/AzureThreshold.java

@@ -2,7 +2,7 @@
  * Orchestration
  * Orchestration is an inference service which provides common additional capabilities for business AI scenarios, such as content filtering and data masking. At the core of the service is the LLM module which allows for an easy, harmonized access to the language models of gen AI hub. The service is designed to be modular and extensible, allowing for the addition of new modules in the future. Each module can be configured independently and at runtime, allowing for a high degree of flexibility in the orchestration of AI services.
  *
- * The version of the OpenAPI document: 0.36.1
+ * The version of the OpenAPI document: 0.43.0
  *
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
@@ -16,10 +16,7 @@
 import com.fasterxml.jackson.annotation.JsonValue;
 import javax.annotation.Nonnull;
 
-/**
- * Threshold for the filter. Setting it to `0` blocks content with low severity, whereas
- * `6` is the most permissive and blocks only the highest severity
- */
+/** Gets or Sets AzureThreshold */
 public enum AzureThreshold {
   NUMBER_0(0),
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/model/ChatDelta.java

@@ -2,7 +2,7 @@
  * Orchestration
  * Orchestration is an inference service which provides common additional capabilities for business AI scenarios, such as content filtering and data masking. At the core of the service is the LLM module which allows for an easy, harmonized access to the language models of gen AI hub. The service is designed to be modular and extensible, allowing for the addition of new modules in the future. Each module can be configured independently and at runtime, allowing for a high degree of flexibility in the orchestration of AI services.
  *
- * The version of the OpenAPI document: 0.36.1
+ * The version of the OpenAPI document: 0.43.0
  *
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).