Merge branch 'main' into chore/refactor-service-binding-loading

Author: MatKuhr

docs/guides/ORCHESTRATION_CHAT_COMPLETION.md

@@ -200,11 +200,11 @@ Change your LLM configuration to add model parameters:
 ```java
 OrchestrationAiModel customGPT4O =
     OrchestrationAiModel.GPT_4O
-        .withModelParams(
+        .withParams(
             Map.of(
                 "max_tokens", 50,
                 "temperature", 0.1,
                 "frequency_penalty", 0,
                 "presence_penalty", 0))
-        .withModelVersion("2024-05-13");
+        .withVersion("2024-05-13");
 ```

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java

@@ -40,9 +40,7 @@ public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
    */
   @Nonnull
   public String getContent() throws OpenAiClientException {
-    if (getChoices().isEmpty()) {
-      return "";
-    }
+    //    We expect choices to be defined and never empty.
     if ("content_filter".equals(getChoices().get(0).getFinishReason())) {
       throw new OpenAiClientException("Content filter filtered the output.");
     }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java

@@ -13,7 +13,7 @@
 @AllArgsConstructor
 public class OrchestrationAiModel {
   /** The name of the model */
-  String modelName;
+  String name;
 
   /**
    * Optional parameters on this model.
@@ -26,10 +26,10 @@ public class OrchestrationAiModel {
    *     "presence_penalty", 0)
    * }</pre>
    */
-  Map<String, Object> modelParams;
+  Map<String, Object> params;
 
   /** The version of the model, defaults to "latest". */
-  String modelVersion;
+  String version;
 
   /** IBM Granite 13B chat completions model */
   public static final OrchestrationAiModel IBM_GRANITE_13B_CHAT =
@@ -106,15 +106,12 @@ public class OrchestrationAiModel {
   public static final OrchestrationAiModel GEMINI_1_5_FLASH =
       new OrchestrationAiModel("gemini-1.5-flash");
 
-  OrchestrationAiModel(@Nonnull final String modelName) {
-    this(modelName, Map.of(), "latest");
+  OrchestrationAiModel(@Nonnull final String name) {
+    this(name, Map.of(), "latest");
   }
 
   @Nonnull
   LLMModuleConfig createConfig() {
-    return new LLMModuleConfig()
-        .modelName(modelName)
-        .modelParams(modelParams)
-        .modelVersion(modelVersion);
+    return new LLMModuleConfig().modelName(name).modelParams(params).modelVersion(version);
   }
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java

@@ -2,8 +2,14 @@
 
 import static lombok.AccessLevel.PACKAGE;
 
+import com.sap.ai.sdk.orchestration.client.model.ChatMessage;
 import com.sap.ai.sdk.orchestration.client.model.CompletionPostResponse;
+import com.sap.ai.sdk.orchestration.client.model.LLMChoice;
 import com.sap.ai.sdk.orchestration.client.model.LLMModuleResultSynchronous;
+import com.sap.ai.sdk.orchestration.client.model.TokenUsage;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
 import javax.annotation.Nonnull;
 import lombok.RequiredArgsConstructor;
 import lombok.Value;
@@ -24,18 +30,47 @@ public class OrchestrationChatResponse {
    */
   @Nonnull
   public String getContent() throws OrchestrationClientException {
-    final var choices =
-        ((LLMModuleResultSynchronous) originalResponse.getOrchestrationResult()).getChoices();
-
-    if (choices.isEmpty()) {
-      return "";
-    }
-
-    final var choice = choices.get(0);
+    final var choice = getCurrentChoice();
 
     if ("content_filter".equals(choice.getFinishReason())) {
       throw new OrchestrationClientException("Content filter filtered the output.");
     }
     return choice.getMessage().getContent();
   }
+
+  /**
+   * Get the token usage.
+   *
+   * @return The token usage.
+   */
+  @Nonnull
+  public TokenUsage getTokenUsage() {
+    return ((LLMModuleResultSynchronous) originalResponse.getOrchestrationResult()).getUsage();
+  }
+
+  /**
+   * Get all messages. This can be used for subsequent prompts as a message history.
+   *
+   * @return A list of all messages.
+   */
+  @Nonnull
+  public List<ChatMessage> getAllMessages() {
+    final var items = Objects.requireNonNull(originalResponse.getModuleResults().getTemplating());
+    final var messages = new ArrayList<>(items);
+    messages.add(getCurrentChoice().getMessage());
+    return messages;
+  }
+
+  /**
+   * Get current choice.
+   *
+   * @return The current choice.
+   */
+  @Nonnull
+  private LLMChoice getCurrentChoice() {
+    //    We expect choices to be defined and never empty.
+    return ((LLMModuleResultSynchronous) originalResponse.getOrchestrationResult())
+        .getChoices()
+        .get(0);
+  }
 }

orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfigTest.java

@@ -69,18 +69,16 @@ void testDpiMaskingConfig() {
   void testLLMConfig() {
     Map<String, Object> params = Map.of("foo", "bar");
     String version = "2024-05-13";
-    OrchestrationAiModel aiModel = GPT_4O.withModelParams(params).withModelVersion(version);
+    OrchestrationAiModel aiModel = GPT_4O.withParams(params).withVersion(version);
     var config = new OrchestrationModuleConfig().withLlmConfig(aiModel);
 
     assertThat(config.getLlmConfig()).isNotNull();
-    assertThat(config.getLlmConfig().getModelName()).isEqualTo(GPT_4O.getModelName());
+    assertThat(config.getLlmConfig().getModelName()).isEqualTo(GPT_4O.getName());
     assertThat(config.getLlmConfig().getModelParams()).isEqualTo(params);
     assertThat(config.getLlmConfig().getModelVersion()).isEqualTo(version);
 
-    assertThat(GPT_4O.getModelParams())
-        .withFailMessage("Static models should be unchanged")
-        .isEmpty();
-    assertThat(GPT_4O.getModelVersion())
+    assertThat(GPT_4O.getParams()).withFailMessage("Static models should be unchanged").isEmpty();
+    assertThat(GPT_4O.getVersion())
         .withFailMessage("Static models should be unchanged")
         .isEqualTo("latest");
   }

orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java

@@ -52,7 +52,7 @@
 @WireMockTest
 class OrchestrationUnitTest {
   static final OrchestrationAiModel CUSTOM_GPT_35 =
-      GPT_35_TURBO_16K.withModelParams(
+      GPT_35_TURBO_16K.withParams(
           Map.of(
               "max_tokens", 50,
               "temperature", 0.1,
@@ -129,9 +129,9 @@ void testTemplating() throws IOException {
 
     final var response = result.getOriginalResponse();
     assertThat(response.getRequestId()).isEqualTo("26ea36b5-c196-4806-a9a6-a686f0c6ad91");
-    assertThat(response.getModuleResults().getTemplating().get(0).getContent())
+    assertThat(result.getAllMessages().get(0).getContent())
         .isEqualTo("Reply with 'Orchestration Service is working!' in German");
-    assertThat(response.getModuleResults().getTemplating().get(0).getRole()).isEqualTo("user");
+    assertThat(result.getAllMessages().get(0).getRole()).isEqualTo("user");
     var llm = (LLMModuleResultSynchronous) response.getModuleResults().getLlm();
     assertThat(llm).isNotNull();
     assertThat(llm.getId()).isEqualTo("chatcmpl-9lzPV4kLrXjFckOp2yY454wksWBoj");
@@ -144,7 +144,7 @@ void testTemplating() throws IOException {
         .isEqualTo("Orchestration Service funktioniert!");
     assertThat(choices.get(0).getMessage().getRole()).isEqualTo("assistant");
     assertThat(choices.get(0).getFinishReason()).isEqualTo("stop");
-    var usage = llm.getUsage();
+    var usage = result.getTokenUsage();
     assertThat(usage.getCompletionTokens()).isEqualTo(7);
     assertThat(usage.getPromptTokens()).isEqualTo(19);
     assertThat(usage.getTotalTokens()).isEqualTo(26);
@@ -159,7 +159,7 @@ void testTemplating() throws IOException {
         .isEqualTo("Orchestration Service funktioniert!");
     assertThat(choices.get(0).getMessage().getRole()).isEqualTo("assistant");
     assertThat(choices.get(0).getFinishReason()).isEqualTo("stop");
-    usage = orchestrationResult.getUsage();
+    usage = result.getTokenUsage();
     assertThat(usage.getCompletionTokens()).isEqualTo(7);
     assertThat(usage.getPromptTokens()).isEqualTo(19);
     assertThat(usage.getTotalTokens()).isEqualTo(26);
@@ -380,17 +380,4 @@ void testErrorHandling() {
 
     softly.assertAll();
   }
-
-  @Test
-  void testEmptyChoicesResponse() {
-    stubFor(
-        post(urlPathEqualTo("/v2/inference/deployments/abcdef0123456789/completion"))
-            .willReturn(
-                aResponse()
-                    .withBodyFile("emptyChoicesResponse.json")
-                    .withHeader("Content-Type", "application/json")));
-    final var result = client.chatCompletion(prompt, config);
-
-    assertThat(result.getContent()).isEmpty();
-  }
 }

orchestration/src/test/resources/__files/emptyChoicesResponse.json

@@ -37,14 +37,14 @@ void testTemplate() {
     assertThat(controller.config.getLlmConfig()).isNotNull();
     final var modelName = controller.config.getLlmConfig().getModelName();
 
-    final var response = controller.template();
-    final var result = response.getOriginalResponse();
+    final var result = controller.template();
+    final var response = result.getOriginalResponse();
 
-    assertThat(result.getRequestId()).isNotEmpty();
-    assertThat(result.getModuleResults().getTemplating().get(0).getContent())
+    assertThat(response.getRequestId()).isNotEmpty();
+    assertThat(result.getAllMessages().get(0).getContent())
         .isEqualTo("Reply with 'Orchestration Service is working!' in German");
-    assertThat(result.getModuleResults().getTemplating().get(0).getRole()).isEqualTo("user");
-    var llm = (LLMModuleResultSynchronous) result.getModuleResults().getLlm();
+    assertThat(result.getAllMessages().get(0).getRole()).isEqualTo("user");
+    var llm = (LLMModuleResultSynchronous) response.getModuleResults().getLlm();
     assertThat(llm.getId()).isNotEmpty();
     assertThat(llm.getObject()).isEqualTo("chat.completion");
     assertThat(llm.getCreated()).isGreaterThan(1);
@@ -54,12 +54,12 @@ void testTemplate() {
     assertThat(choices.get(0).getMessage().getContent()).isNotEmpty();
     assertThat(choices.get(0).getMessage().getRole()).isEqualTo("assistant");
     assertThat(choices.get(0).getFinishReason()).isEqualTo("stop");
-    var usage = llm.getUsage();
+    var usage = result.getTokenUsage();
     assertThat(usage.getCompletionTokens()).isGreaterThan(1);
     assertThat(usage.getPromptTokens()).isGreaterThan(1);
     assertThat(usage.getTotalTokens()).isGreaterThan(1);
 
-    var orchestrationResult = ((LLMModuleResultSynchronous) result.getOrchestrationResult());
+    var orchestrationResult = ((LLMModuleResultSynchronous) response.getOrchestrationResult());
     assertThat(orchestrationResult.getObject()).isEqualTo("chat.completion");
     assertThat(orchestrationResult.getCreated()).isGreaterThan(1);
     assertThat(orchestrationResult.getModel()).isEqualTo(modelName);
@@ -68,7 +68,7 @@ void testTemplate() {
     assertThat(choices.get(0).getMessage().getContent()).isNotEmpty();
     assertThat(choices.get(0).getMessage().getRole()).isEqualTo("assistant");
     assertThat(choices.get(0).getFinishReason()).isEqualTo("stop");
-    usage = orchestrationResult.getUsage();
+    usage = result.getTokenUsage();
     assertThat(usage.getCompletionTokens()).isGreaterThan(1);
     assertThat(usage.getPromptTokens()).isGreaterThan(1);
     assertThat(usage.getTotalTokens()).isGreaterThan(1);