feat: Spring AI🍃Tool Calling (#320)

* Spring AI functions

* Test function callback

* Remove test

* First tool call

* Best effort

* TODO

* Formatting

* Fix test

* Removed TODO

* Checkstyle

* lombok

* List of tool calls supported

* Unit test wip

* Unit test almost

* Unit test finished

* Green

* Formatting

* Green

* new ToolCall class

* Added documentation and release notes

* Replaced ToolCall class with existing ResponseMessageToolCall

* Tool call wip

* Spring AI 1.0.0-M6

* Reduced options class size

* Added tests

* Updated release notes

* Replaced FunctionCallbacks with ToolCallbacks

* Updated docs

* Updated docs

* Removed useless code

* Removed whitespace

* Added deprecation notice

* Update orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModel.java

Co-authored-by: Alexander Dümont <[email protected]>

* composition for toolCallingManager

* since 1.4.0

* dependencies

* make fields final again

---------

Co-authored-by: SAP Cloud SDK Bot <[email protected]>
Co-authored-by: Alexander Dümont <[email protected]>
Co-authored-by: Alexander Dümont <[email protected]>

Author: 4 people

docs/guides/SPRING_AI_INTEGRATION.md

@@ -5,6 +5,8 @@
 - [Introduction](#introduction)
 - [Orchestration Chat Completion](#orchestration-chat-completion)
 - [Orchestration Masking](#orchestration-masking)
+- [Stream chat completion](#stream-chat-completion)
+- [Tool Calling](#tool-calling)
 
 ## Introduction
 
@@ -32,7 +34,7 @@ First, add the Spring AI dependency to your `pom.xml`:
 
 :::note Spring AI Milestone Version
 Note that currently no stable version of Spring AI exists just yet.
-The AI SDK currently uses the [M5 milestone](https://spring.io/blog/2024/12/23/spring-ai-1-0-0-m5-released).
+The AI SDK currently uses the [M6 milestone](https://spring.io/blog/2025/02/14/spring-ai-1-0-0-m6-released).
 
 Please be aware that future versions of the AI SDK may increase the Spring AI version.
 :::
@@ -99,3 +101,40 @@ Flux<String> responseFlux =
 _Note: A Spring endpoint can return `Flux` instead of `ResponseEntity`._
 
 Please find [an example in our Spring Boot application](../../sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/SpringAiOrchestrationService.java).
+
+## Tool Calling
+
+First define a function that will be called by the LLM:
+
+```java
+class WeatherMethod {
+  enum Unit {C,F}
+  record Request(String location, Unit unit) {}
+  record Response(double temp, Unit unit) {}
+
+  @Tool(description = "Get the weather in location")
+  Response getCurrentWeather(@ToolParam Request request) {
+    int temperature = request.location.hashCode() % 30;
+    return new Response(temperature, request.unit);
+  }
+}
+```
+
+Then add your tool to the options:
+
+```java
+ChatModel client = new OrchestrationChatModel();
+OrchestrationModuleConfig config = new OrchestrationModuleConfig().withLlmConfig(GPT_35_TURBO);
+OrchestrationChatOptions opts = new OrchestrationChatOptions(config);
+
+options.setToolCallbacks(List.of(ToolCallbacks.from(new WeatherMethod())));
+
+options.setInternalToolExecutionEnabled(false);// tool execution is not yet available in orchestration
+
+Prompt prompt = new Prompt("What is the weather in Potsdam and in Toulouse?", options);
+
+ChatResponse response = client.call(prompt);
+```
+
+Please find [an example in our Spring Boot application](../../sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/SpringAiOrchestrationService.java).
+

docs/release-notes/release_notes.md

@@ -12,7 +12,7 @@
 
 ### ✨ New Functionality
 
--
+- [Add Spring AI tool calling](../guides/SPRING_AI_INTEGRATION.md#tool-calling).
 
 ### 📈 Improvements
 

orchestration/pom.xml

@@ -31,11 +31,11 @@
   </developers>
   <properties>
     <project.rootdir>${project.basedir}/../</project.rootdir>
-    <coverage.complexity>80%</coverage.complexity>
+    <coverage.complexity>81%</coverage.complexity>
     <coverage.line>92%</coverage.line>
     <coverage.instruction>93%</coverage.instruction>
-    <coverage.branch>71%</coverage.branch>
-    <coverage.method>95%</coverage.method>
+    <coverage.branch>74%</coverage.branch>
+    <coverage.method>92%</coverage.method>
     <coverage.class>100%</coverage.class>
   </properties>
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/AssistantMessage.java

@@ -1,14 +1,20 @@
 package com.sap.ai.sdk.orchestration;
 
 import com.google.common.annotations.Beta;
+import com.sap.ai.sdk.orchestration.model.ChatMessage;
+import com.sap.ai.sdk.orchestration.model.ResponseMessageToolCall;
+import com.sap.ai.sdk.orchestration.model.SingleChatMessage;
 import java.util.List;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import lombok.Getter;
 import lombok.Value;
 import lombok.experimental.Accessors;
+import lombok.val;
 
 /** Represents a chat message as 'assistant' to the orchestration service. */
 @Value
+@Getter
 @Accessors(fluent = true)
 public class AssistantMessage implements Message {
 
@@ -20,12 +26,38 @@ public class AssistantMessage implements Message {
   @Getter(onMethod_ = @Beta)
   MessageContent content;
 
+  /** Tool call if there is any. */
+  @Nullable List<ResponseMessageToolCall> toolCalls;
+
   /**
    * Creates a new assistant message with the given single message.
    *
    * @param singleMessage the single message.
    */
   public AssistantMessage(@Nonnull final String singleMessage) {
     content = new MessageContent(List.of(new TextItem(singleMessage)));
+    toolCalls = null;
+  }
+
+  /**
+   * Creates a new assistant message with the given tool calls.
+   *
+   * @param toolCalls list of tool call objects
+   */
+  public AssistantMessage(@Nonnull final List<ResponseMessageToolCall> toolCalls) {
+    content = new MessageContent(List.of());
+    this.toolCalls = toolCalls;
+  }
+
+  @Nonnull
+  @Override
+  public ChatMessage createChatMessage() {
+    if (toolCalls() != null) {
+      // content shouldn't be required for tool calls 🤷
+      val message = SingleChatMessage.create().role(role).content("");
+      message.setCustomField("tool_calls", toolCalls);
+      return message;
+    }
+    return Message.super.createChatMessage();
   }
 }

orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java

@@ -8,7 +8,6 @@
 import com.sap.ai.sdk.orchestration.model.TemplatingModuleConfig;
 import io.vavr.control.Option;
 import java.util.ArrayList;
-import java.util.List;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.AccessLevel;
@@ -40,24 +39,28 @@ static CompletionPostRequest toCompletionPostRequest(
 
   @Nonnull
   static TemplatingModuleConfig toTemplateModuleConfig(
-      @Nonnull final OrchestrationPrompt prompt, @Nullable final TemplatingModuleConfig template) {
+      @Nonnull final OrchestrationPrompt prompt, @Nullable final TemplatingModuleConfig config) {
     /*
      * Currently, we have to merge the prompt into the template configuration.
      * This works around the limitation that the template config is required.
      * This comes at the risk that the prompt unintentionally contains the templating pattern "{{? .. }}".
      * In this case, the request will fail, since the templating module will try to resolve the parameter.
      * To be fixed with https://github.tools.sap/AI/llm-orchestration/issues/662
      */
-    val messages = template instanceof Template t ? t.getTemplate() : List.<ChatMessage>of();
-    val responseFormat = template instanceof Template t ? t.getResponseFormat() : null;
+    val template = config instanceof Template t ? t : Template.create().template();
+    val messages = template.getTemplate();
+    val responseFormat = template.getResponseFormat();
     val messagesWithPrompt = new ArrayList<>(messages);
     messagesWithPrompt.addAll(
         prompt.getMessages().stream().map(Message::createChatMessage).toList());
     if (messagesWithPrompt.isEmpty()) {
       throw new IllegalStateException(
           "A prompt is required. Pass at least one message or configure a template with messages or a template reference.");
     }
-    return Template.create().template(messagesWithPrompt).responseFormat(responseFormat);
+    return Template.create()
+        .template(messagesWithPrompt)
+        .tools(template.getTools())
+        .responseFormat(responseFormat);
   }
 
   @Nonnull

orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java

@@ -13,7 +13,7 @@
 import javax.annotation.Nonnull;
 
 /** Interface representing convenience wrappers of chat message to the orchestration service. */
-public sealed interface Message permits UserMessage, AssistantMessage, SystemMessage {
+public sealed interface Message permits AssistantMessage, SystemMessage, ToolMessage, UserMessage {
 
   /**
    * A convenience method to create a user message from a string.

orchestration/src/main/java/com/sap/ai/sdk/orchestration/ToolMessage.java

@@ -0,0 +1,39 @@
+package com.sap.ai.sdk.orchestration;
+
+import com.sap.ai.sdk.orchestration.model.ChatMessage;
+import com.sap.ai.sdk.orchestration.model.SingleChatMessage;
+import java.util.List;
+import javax.annotation.Nonnull;
+import lombok.Value;
+import lombok.experimental.Accessors;
+
+/**
+ * Represents a chat message as 'tool' to the orchestration service.
+ *
+ * @since 1.4.0
+ */
+@Value
+@Accessors(fluent = true)
+public class ToolMessage implements Message {
+
+  /** The role of the assistant. */
+  @Nonnull String role = "tool";
+
+  @Nonnull String id;
+
+  @Nonnull String content;
+
+  @Nonnull
+  @Override
+  public MessageContent content() {
+    return new MessageContent(List.of(new TextItem(content)));
+  }
+
+  @Nonnull
+  @Override
+  public ChatMessage createChatMessage() {
+    final SingleChatMessage message = SingleChatMessage.create().role(role()).content(content);
+    message.setCustomField("tool_call_id", id);
+    return message;
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/spring/OrchestrationChatModel.java

@@ -1,25 +1,32 @@
 package com.sap.ai.sdk.orchestration.spring;
 
 import static com.sap.ai.sdk.orchestration.OrchestrationClient.toCompletionPostRequest;
+import static com.sap.ai.sdk.orchestration.model.ResponseMessageToolCall.TypeEnum.FUNCTION;
 
 import com.google.common.annotations.Beta;
 import com.sap.ai.sdk.orchestration.AssistantMessage;
 import com.sap.ai.sdk.orchestration.OrchestrationChatCompletionDelta;
 import com.sap.ai.sdk.orchestration.OrchestrationClient;
 import com.sap.ai.sdk.orchestration.OrchestrationPrompt;
 import com.sap.ai.sdk.orchestration.SystemMessage;
+import com.sap.ai.sdk.orchestration.ToolMessage;
 import com.sap.ai.sdk.orchestration.UserMessage;
+import com.sap.ai.sdk.orchestration.model.ResponseMessageToolCall;
+import com.sap.ai.sdk.orchestration.model.ResponseMessageToolCallFunction;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 import javax.annotation.Nonnull;
-import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
+import org.springframework.ai.chat.messages.AssistantMessage.ToolCall;
 import org.springframework.ai.chat.messages.Message;
+import org.springframework.ai.chat.messages.ToolResponseMessage;
 import org.springframework.ai.chat.model.ChatModel;
 import org.springframework.ai.chat.model.ChatResponse;
 import org.springframework.ai.chat.prompt.Prompt;
+import org.springframework.ai.model.tool.DefaultToolCallingManager;
+import org.springframework.ai.model.tool.ToolCallingChatOptions;
 import reactor.core.publisher.Flux;
 
 /**
@@ -29,28 +36,48 @@
  */
 @Beta
 @Slf4j
-@RequiredArgsConstructor
 public class OrchestrationChatModel implements ChatModel {
-  @Nonnull private OrchestrationClient client;
+  @Nonnull private final OrchestrationClient client;
+
+  @Nonnull
+  private final DefaultToolCallingManager toolCallingManager =
+      DefaultToolCallingManager.builder().build();
 
   /**
    * Default constructor.
    *
    * @since 1.2.0
    */
   public OrchestrationChatModel() {
-    this.client = new OrchestrationClient();
+    this(new OrchestrationClient());
+  }
+
+  /**
+   * Constructor with a custom client.
+   *
+   * @since 1.2.0
+   */
+  public OrchestrationChatModel(@Nonnull final OrchestrationClient client) {
+    this.client = client;
   }
 
   @Nonnull
   @Override
   public ChatResponse call(@Nonnull final Prompt prompt) {
-
     if (prompt.getOptions() instanceof OrchestrationChatOptions options) {
 
       val orchestrationPrompt = toOrchestrationPrompt(prompt);
-      val response = client.chatCompletion(orchestrationPrompt, options.getConfig());
-      return new OrchestrationSpringChatResponse(response);
+      val response =
+          new OrchestrationSpringChatResponse(
+              client.chatCompletion(orchestrationPrompt, options.getConfig()));
+
+      if (ToolCallingChatOptions.isInternalToolExecutionEnabled(prompt.getOptions())
+          && response.hasToolCalls()) {
+        val toolExecutionResult = toolCallingManager.executeToolCalls(prompt, response);
+        // Send the tool execution result back to the model.
+        return call(new Prompt(toolExecutionResult.conversationHistory(), prompt.getOptions()));
+      }
+      return response;
     }
     throw new IllegalArgumentException(
         "Please add OrchestrationChatOptions to the Prompt: new Prompt(\"message\", new OrchestrationChatOptions(config))");
@@ -92,18 +119,47 @@ private OrchestrationPrompt toOrchestrationPrompt(@Nonnull final Prompt prompt)
   @Nonnull
   private static com.sap.ai.sdk.orchestration.Message[] toOrchestrationMessages(
       @Nonnull final List<Message> messages) {
-    final Function<Message, com.sap.ai.sdk.orchestration.Message> mapper =
+    final Function<Message, List<com.sap.ai.sdk.orchestration.Message>> mapper =
         msg ->
             switch (msg.getMessageType()) {
               case SYSTEM:
-                yield new SystemMessage(msg.getText());
+                yield List.of(new SystemMessage(msg.getText()));
               case USER:
-                yield new UserMessage(msg.getText());
+                yield List.of(new UserMessage(msg.getText()));
               case ASSISTANT:
-                yield new AssistantMessage(msg.getText());
+                val springToolCalls =
+                    ((org.springframework.ai.chat.messages.AssistantMessage) msg).getToolCalls();
+                if (springToolCalls != null) {
+                  final List<ResponseMessageToolCall> sdkToolCalls =
+                      springToolCalls.stream()
+                          .map(OrchestrationChatModel::toOrchestrationToolCall)
+                          .toList();
+                  yield List.of(new AssistantMessage(sdkToolCalls));
+                }
+                yield List.of(new AssistantMessage(msg.getText()));
               case TOOL:
-                throw new IllegalArgumentException("Tool messages are not supported");
+                val toolResponses = ((ToolResponseMessage) msg).getResponses();
+                yield toolResponses.stream()
+                    .map(
+                        r ->
+                            (com.sap.ai.sdk.orchestration.Message)
+                                new ToolMessage(r.id(), r.responseData()))
+                    .toList();
             };
-    return messages.stream().map(mapper).toArray(com.sap.ai.sdk.orchestration.Message[]::new);
+    return messages.stream()
+        .map(mapper)
+        .flatMap(List::stream)
+        .toArray(com.sap.ai.sdk.orchestration.Message[]::new);
+  }
+
+  @Nonnull
+  private static ResponseMessageToolCall toOrchestrationToolCall(@Nonnull final ToolCall toolCall) {
+    return ResponseMessageToolCall.create()
+        .id(toolCall.id())
+        .type(FUNCTION)
+        .function(
+            ResponseMessageToolCallFunction.create()
+                .name(toolCall.name())
+                .arguments(toolCall.arguments()));
   }
 }