Merge remote-tracking branch 'origin/main' into feat/orch-convenient-filtering

Author: rpanackal

.github/workflows/perform-release.yml

@@ -88,9 +88,6 @@ jobs:
         with:
           distribution: "sapmachine"
           java-version: ${{ env.JAVA_VERSION }}
-          server-id: ossrh
-          server-username: MAVEN_CENTRAL_USER # env variable for username in deploy
-          server-password: MAVEN_CENTRAL_PASSWORD # env variable for token in deploy
 
       - name: "Download Release Asset"
         id: download-asset
@@ -113,8 +110,8 @@ jobs:
 
       - name: "Deploy"
         run: |
-          MVN_ARGS="${{ env.MVN_CLI_ARGS }} deploy -Drelease -s settings.xml"
-          mvn $MVN_ARGS
+          MVN_ARGS="${{ env.MVN_CLI_ARGS }} -Drelease -s settings.xml"
+          mvn deploy $MVN_ARGS
         env:
           MAVEN_GPG_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
 

docs/guides/ORCHESTRATION_CHAT_COMPLETION.md

@@ -91,8 +91,7 @@ var prompt = new OrchestrationPrompt("Hello world! Why is this phrase so famous?
 
 var result = client.chatCompletion(prompt, config);
 
-String messageResult =
-        result.getOrchestrationResult().getChoices().get(0).getMessage().getContent();
+String messageResult = result.getContent();
 ```
 
 In this example, the Orchestration service generates a response to the user message "Hello world! Why is this phrase so famous?".

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationChatResponse.java

@@ -0,0 +1,41 @@
+package com.sap.ai.sdk.orchestration;
+
+import static lombok.AccessLevel.PACKAGE;
+
+import com.sap.ai.sdk.orchestration.client.model.CompletionPostResponse;
+import com.sap.ai.sdk.orchestration.client.model.LLMModuleResultSynchronous;
+import javax.annotation.Nonnull;
+import lombok.RequiredArgsConstructor;
+import lombok.Value;
+
+/** Orchestration chat completion output. */
+@Value
+@RequiredArgsConstructor(access = PACKAGE)
+public class OrchestrationChatResponse {
+  CompletionPostResponse originalResponse;
+
+  /**
+   * Get the message content from the output.
+   *
+   * <p>Note: If there are multiple choices only the first one is returned
+   *
+   * @return the message content or empty string.
+   * @throws OrchestrationClientException if the content filter filtered the output.
+   */
+  @Nonnull
+  public String getContent() throws OrchestrationClientException {
+    final var choices =
+        ((LLMModuleResultSynchronous) originalResponse.getOrchestrationResult()).getChoices();
+
+    if (choices.isEmpty()) {
+      return "";
+    }
+
+    final var choice = choices.get(0);
+
+    if ("content_filter".equals(choice.getFinishReason())) {
+      throw new OrchestrationClientException("Content filter filtered the output.");
+    }
+    return choice.getMessage().getContent();
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java

@@ -101,12 +101,12 @@ public static CompletionPostRequest toCompletionPostRequest(
    * @throws OrchestrationClientException if the request fails.
    */
   @Nonnull
-  public CompletionPostResponse chatCompletion(
+  public OrchestrationChatResponse chatCompletion(
       @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config)
       throws OrchestrationClientException {
 
     val request = toCompletionPostRequest(prompt, config);
-    return executeRequest(request);
+    return new OrchestrationChatResponse(executeRequest(request));
   }
 
   /**

orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java

@@ -116,8 +116,7 @@ void testCompletion() {
     final var result = client.chatCompletion(prompt, config);
 
     assertThat(result).isNotNull();
-    var orchestrationResult = (LLMModuleResultSynchronous) result.getOrchestrationResult();
-    assertThat(orchestrationResult.getChoices().get(0).getMessage().getContent()).isNotEmpty();
+    assertThat(result.getContent()).isNotEmpty();
   }
 
   @Test
@@ -136,11 +135,12 @@ void testTemplating() throws IOException {
     final var result =
         client.chatCompletion(new OrchestrationPrompt(inputParams, template), config);
 
-    assertThat(result.getRequestId()).isEqualTo("26ea36b5-c196-4806-a9a6-a686f0c6ad91");
-    assertThat(result.getModuleResults().getTemplating().get(0).getContent())
+    final var response = result.getOriginalResponse();
+    assertThat(response.getRequestId()).isEqualTo("26ea36b5-c196-4806-a9a6-a686f0c6ad91");
+    assertThat(response.getModuleResults().getTemplating().get(0).getContent())
         .isEqualTo("Reply with 'Orchestration Service is working!' in German");
-    assertThat(result.getModuleResults().getTemplating().get(0).getRole()).isEqualTo("user");
-    var llm = (LLMModuleResultSynchronous) result.getModuleResults().getLlm();
+    assertThat(response.getModuleResults().getTemplating().get(0).getRole()).isEqualTo("user");
+    var llm = (LLMModuleResultSynchronous) response.getModuleResults().getLlm();
     assertThat(llm.getId()).isEqualTo("chatcmpl-9lzPV4kLrXjFckOp2yY454wksWBoj");
     assertThat(llm.getObject()).isEqualTo("chat.completion");
     assertThat(llm.getCreated()).isEqualTo(1721224505);
@@ -155,7 +155,7 @@ void testTemplating() throws IOException {
     assertThat(usage.getCompletionTokens()).isEqualTo(7);
     assertThat(usage.getPromptTokens()).isEqualTo(19);
     assertThat(usage.getTotalTokens()).isEqualTo(26);
-    var orchestrationResult = (LLMModuleResultSynchronous) result.getOrchestrationResult();
+    var orchestrationResult = (LLMModuleResultSynchronous) response.getOrchestrationResult();
     assertThat(orchestrationResult.getId()).isEqualTo("chatcmpl-9lzPV4kLrXjFckOp2yY454wksWBoj");
     assertThat(orchestrationResult.getObject()).isEqualTo("chat.completion");
     assertThat(orchestrationResult.getCreated()).isEqualTo(1721224505);
@@ -274,7 +274,8 @@ void messagesHistory() throws IOException {
 
     final var result = client.chatCompletion(prompt, config);
 
-    assertThat(result.getRequestId()).isEqualTo("26ea36b5-c196-4806-a9a6-a686f0c6ad91");
+    assertThat(result.getOriginalResponse().getRequestId())
+        .isEqualTo("26ea36b5-c196-4806-a9a6-a686f0c6ad91");
 
     // verify that the history is sent correctly
     try (var requestInputStream = fileLoader.apply("messagesHistoryRequest.json")) {
@@ -298,13 +299,13 @@ void maskingPseudonymization() throws IOException {
         createMaskingConfig(DPIConfig.MethodEnum.PSEUDONYMIZATION, DPIEntities.PHONE);
 
     final var result = client.chatCompletion(prompt, config.withMaskingConfig(maskingConfig));
+    final var response = result.getOriginalResponse();
 
-    assertThat(result).isNotNull();
-    GenericModuleResult inputMasking = result.getModuleResults().getInputMasking();
+    assertThat(response).isNotNull();
+    GenericModuleResult inputMasking = response.getModuleResults().getInputMasking();
     assertThat(inputMasking.getMessage()).isEqualTo("Input to LLM is masked successfully.");
     assertThat(inputMasking.getData()).isNotNull();
-    final var choices = ((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices();
-    assertThat(choices.get(0).getMessage().getContent()).contains("Hi Mallory");
+    assertThat(result.getContent()).contains("Hi Mallory");
 
     // verify that the request is sent correctly
     try (var requestInputStream = fileLoader.apply("maskingRequest.json")) {
@@ -400,4 +401,17 @@ void testErrorHandling() {
 
     softly.assertAll();
   }
+
+  @Test
+  void testEmptyChoicesResponse() {
+    stubFor(
+        post(urlPathEqualTo("/v2/inference/deployments/abcdef0123456789/completion"))
+            .willReturn(
+                aResponse()
+                    .withBodyFile("emptyChoicesResponse.json")
+                    .withHeader("Content-Type", "application/json")));
+    final var result = client.chatCompletion(prompt, config);
+
+    assertThat(result.getContent()).isEmpty();
+  }
 }

orchestration/src/test/resources/__files/emptyChoicesResponse.json

@@ -0,0 +1,35 @@
+{
+  "request_id": "26ea36b5-c196-4806-a9a6-a686f0c6ad91",
+  "module_results": {
+    "templating": [
+      {
+        "role": "user",
+        "content": "Reply with 'Orchestration Service is working!' in German"
+      }
+    ],
+    "llm": {
+      "id": "chatcmpl-9lzPV4kLrXjFckOp2yY454wksWBoj",
+      "object": "chat.completion",
+      "created": 1721224505,
+      "model": "gpt-35-turbo-16k",
+      "choices": [],
+      "usage": {
+        "completion_tokens": 7,
+        "prompt_tokens": 19,
+        "total_tokens": 26
+      }
+    }
+  },
+  "orchestration_result": {
+    "id": "chatcmpl-9lzPV4kLrXjFckOp2yY454wksWBoj",
+    "object": "chat.completion",
+    "created": 1721224505,
+    "model": "gpt-35-turbo-16k",
+    "choices": [],
+    "usage": {
+      "completion_tokens": 7,
+      "prompt_tokens": 19,
+      "total_tokens": 26
+    }
+  }
+}

pom.xml

@@ -724,6 +724,28 @@ https://gitbox.apache.org/repos/asf?p=maven-pmd-plugin.git;a=blob_plain;f=src/ma
           </plugins>
         </pluginManagement>
         <plugins>
+          <!-- The release artifacts don't contain our custom config files for these plugins -->
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-checkstyle-plugin</artifactId>
+            <configuration>
+              <skip>true</skip>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-pmd-plugin</artifactId>
+            <configuration>
+              <skip>true</skip>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>com.github.spotbugs</groupId>
+            <artifactId>spotbugs-maven-plugin</artifactId>
+            <configuration>
+              <skip>true</skip>
+            </configuration>
+          </plugin>
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-source-plugin</artifactId>

sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java

@@ -1,12 +1,12 @@
 package com.sap.ai.sdk.app.controllers;
 
+import com.sap.ai.sdk.orchestration.OrchestrationChatResponse;
 import com.sap.ai.sdk.orchestration.OrchestrationClient;
 import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig;
 import com.sap.ai.sdk.orchestration.OrchestrationPrompt;
 import com.sap.ai.sdk.orchestration.client.model.AzureContentSafety;
 import com.sap.ai.sdk.orchestration.client.model.AzureThreshold;
 import com.sap.ai.sdk.orchestration.client.model.ChatMessage;
-import com.sap.ai.sdk.orchestration.client.model.CompletionPostResponse;
 import com.sap.ai.sdk.orchestration.client.model.DPIConfig;
 import com.sap.ai.sdk.orchestration.client.model.DPIEntities;
 import com.sap.ai.sdk.orchestration.client.model.DPIEntityConfig;
@@ -40,7 +40,7 @@ class OrchestrationController {
    */
   @GetMapping("/completion")
   @Nonnull
-  public CompletionPostResponse completion() {
+  public OrchestrationChatResponse completion() {
     final var prompt = new OrchestrationPrompt("Hello world! Why is this phrase so famous?");
 
     return client.chatCompletion(prompt, config);
@@ -53,7 +53,7 @@ public CompletionPostResponse completion() {
    */
   @GetMapping("/template")
   @Nonnull
-  public CompletionPostResponse template() {
+  public OrchestrationChatResponse template() {
     final var template =
         new ChatMessage()
             .role("user")
@@ -74,7 +74,7 @@ public CompletionPostResponse template() {
    */
   @GetMapping("/messagesHistory")
   @Nonnull
-  public CompletionPostResponse messagesHistory() {
+  public OrchestrationChatResponse messagesHistory() {
     final List<ChatMessage> messagesHistory =
         List.of(
             new ChatMessage().role("user").content("What is the capital of France?"),
@@ -94,7 +94,7 @@ public CompletionPostResponse messagesHistory() {
    */
   @GetMapping("/filter/{threshold}")
   @Nonnull
-  public CompletionPostResponse filter(
+  public OrchestrationChatResponse filter(
       @Nonnull @PathVariable("threshold") final AzureThreshold threshold) {
     final var prompt =
         new OrchestrationPrompt(
@@ -125,7 +125,7 @@ public CompletionPostResponse filter(
    */
   @GetMapping("/maskingAnonymization")
   @Nonnull
-  public CompletionPostResponse maskingAnonymization() {
+  public OrchestrationChatResponse maskingAnonymization() {
     final var systemMessage =
         new ChatMessage()
             .role("system")
@@ -156,7 +156,7 @@ public CompletionPostResponse maskingAnonymization() {
    */
   @GetMapping("/maskingPseudonymization")
   @Nonnull
-  public CompletionPostResponse maskingPseudonymization() {
+  public OrchestrationChatResponse maskingPseudonymization() {
     final var systemMessage =
         new ChatMessage()
             .role("system")

sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java

@@ -27,18 +27,13 @@ void testCompletion() {
     final var result = controller.completion();
 
     assertThat(result).isNotNull();
-    assertThat(
-            ((LLMModuleResultSynchronous) result.getOrchestrationResult())
-                .getChoices()
-                .get(0)
-                .getMessage()
-                .getContent())
-        .isNotEmpty();
+    assertThat(result.getContent()).isNotEmpty();
   }
 
   @Test
   void testTemplate() {
-    final var result = controller.template();
+    final var response = controller.template();
+    final var result = response.getOriginalResponse();
 
     assertThat(result.getRequestId()).isNotEmpty();
     assertThat(result.getModuleResults().getTemplating().get(0).getContent())
@@ -64,20 +59,21 @@ void testTemplate() {
     assertThat(orchestrationResult.getCreated()).isGreaterThan(1);
     assertThat(orchestrationResult.getModel())
         .isEqualTo(OrchestrationController.LLM_CONFIG.getModelName());
-    choices = ((LLMModuleResultSynchronous) orchestrationResult).getChoices();
+    choices = orchestrationResult.getChoices();
     assertThat(choices.get(0).getIndex()).isZero();
     assertThat(choices.get(0).getMessage().getContent()).isNotEmpty();
     assertThat(choices.get(0).getMessage().getRole()).isEqualTo("assistant");
     assertThat(choices.get(0).getFinishReason()).isEqualTo("stop");
-    usage = ((LLMModuleResultSynchronous) orchestrationResult).getUsage();
+    usage = orchestrationResult.getUsage();
     assertThat(usage.getCompletionTokens()).isGreaterThan(1);
     assertThat(usage.getPromptTokens()).isGreaterThan(1);
     assertThat(usage.getTotalTokens()).isGreaterThan(1);
   }
 
   @Test
   void testLenientContentFilter() {
-    var result = controller.filter(AzureThreshold.NUMBER_4);
+    var response = controller.filter(AzureThreshold.NUMBER_4);
+    var result = response.getOriginalResponse();
     var llmChoice =
         ((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices().get(0);
     assertThat(llmChoice.getFinishReason()).isEqualTo("stop");
@@ -97,15 +93,16 @@ void testStrictContentFilter() {
 
   @Test
   void testMessagesHistory() {
-    CompletionPostResponse result = controller.messagesHistory();
+    CompletionPostResponse result = controller.messagesHistory().getOriginalResponse();
     final var choices = ((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices();
     assertThat(choices.get(0).getMessage().getContent()).isNotEmpty();
   }
 
   @SuppressWarnings("unchecked")
   @Test
   void testMaskingAnonymization() {
-    var result = controller.maskingAnonymization();
+    var response = controller.maskingAnonymization();
+    var result = response.getOriginalResponse();
     var llmChoice =
         ((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices().get(0);
     assertThat(llmChoice.getFinishReason()).isEqualTo("stop");
@@ -124,7 +121,8 @@ void testMaskingAnonymization() {
   @SuppressWarnings("unchecked")
   @Test
   void testMaskingPseudonymization() {
-    var result = controller.maskingPseudonymization();
+    var response = controller.maskingPseudonymization();
+    var result = response.getOriginalResponse();
     var llmChoice =
         ((LLMModuleResultSynchronous) result.getOrchestrationResult()).getChoices().get(0);
     assertThat(llmChoice.getFinishReason()).isEqualTo("stop");