Micrometer instrumentation for tool calling

Introduce instrumentation for tool calling using the Micrometer Observation API.

By default, metadata about tool calling are exported as metrics and traces.
Optionally, the actual tool call input and result can be exported as well by enabling the dedicated feature flag.

Signed-off-by: Thomas Vitale <[email protected]>
Signed-off-by: minsoo.nam <[email protected]>

Author: ThomasVitale

auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfiguration.java

@@ -16,17 +16,17 @@
 
 package org.springframework.ai.model.tool.autoconfigure;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import io.micrometer.observation.ObservationRegistry;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.ai.chat.model.ChatModel;
 import org.springframework.ai.model.tool.ToolCallingManager;
 import org.springframework.ai.tool.ToolCallback;
 import org.springframework.ai.tool.ToolCallbackProvider;
 import org.springframework.ai.tool.execution.DefaultToolExecutionExceptionProcessor;
 import org.springframework.ai.tool.execution.ToolExecutionExceptionProcessor;
+import org.springframework.ai.tool.observation.ToolCallingContentObservationFilter;
+import org.springframework.ai.tool.observation.ToolCallingObservationConvention;
 import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver;
 import org.springframework.ai.tool.resolution.SpringBeanToolCallbackResolver;
 import org.springframework.ai.tool.resolution.StaticToolCallbackResolver;
@@ -35,9 +35,14 @@
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.support.GenericApplicationContext;
 
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * Auto-configuration for common tool calling features of {@link ChatModel}.
  *
@@ -47,8 +52,11 @@
  */
 @AutoConfiguration
 @ConditionalOnClass(ChatModel.class)
+@EnableConfigurationProperties(ToolCallingProperties.class)
 public class ToolCallingAutoConfiguration {
 
+	private static final Logger logger = LoggerFactory.getLogger(ToolCallingAutoConfiguration.class);
+
 	@Bean
 	@ConditionalOnMissingBean
 	ToolCallbackResolver toolCallbackResolver(GenericApplicationContext applicationContext,
@@ -76,12 +84,27 @@ ToolExecutionExceptionProcessor toolExecutionExceptionProcessor() {
 	@ConditionalOnMissingBean
 	ToolCallingManager toolCallingManager(ToolCallbackResolver toolCallbackResolver,
 			ToolExecutionExceptionProcessor toolExecutionExceptionProcessor,
-			ObjectProvider<ObservationRegistry> observationRegistry) {
-		return ToolCallingManager.builder()
+			ObjectProvider<ObservationRegistry> observationRegistry,
+			ObjectProvider<ToolCallingObservationConvention> observationConvention) {
+		var toolCallingManager = ToolCallingManager.builder()
 			.observationRegistry(observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP))
 			.toolCallbackResolver(toolCallbackResolver)
 			.toolExecutionExceptionProcessor(toolExecutionExceptionProcessor)
 			.build();
+
+		observationConvention.ifAvailable(toolCallingManager::setObservationConvention);
+
+		return toolCallingManager;
+	}
+
+	@Bean
+	@ConditionalOnMissingBean
+	@ConditionalOnProperty(prefix = ToolCallingProperties.CONFIG_PREFIX + ".observations", name = "include-content",
+			havingValue = "true")
+	ToolCallingContentObservationFilter toolCallingContentObservationFilter() {
+		logger.warn(
+				"You have enabled the inclusion of the tool call arguments and result in the observations, with the risk of exposing sensitive or private information. Please, be careful!");
+		return new ToolCallingContentObservationFilter();
 	}
 
 }

auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/main/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingProperties.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2023-2025 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.model.tool.autoconfigure;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+/**
+ * Configuration properties for tool calling.
+ *
+ * @author Thomas Vitale
+ * @since 1.0.0
+ */
+@ConfigurationProperties(ToolCallingProperties.CONFIG_PREFIX)
+public class ToolCallingProperties {
+
+	public static final String CONFIG_PREFIX = "spring.ai.tools";
+
+	private final Observations observations = new Observations();
+
+	public static class Observations {
+
+		/**
+		 * Whether to include the tool call content in the observations.
+		 */
+		private boolean includeContent = false;
+
+		public boolean isIncludeContent() {
+			return includeContent;
+		}
+
+		public void setIncludeContent(boolean includeContent) {
+			this.includeContent = includeContent;
+		}
+
+	}
+
+}

auto-configurations/models/tool/spring-ai-autoconfigure-model-tool/src/test/java/org/springframework/ai/model/tool/autoconfigure/ToolCallingAutoConfigurationTests.java

@@ -31,6 +31,7 @@
 import org.springframework.ai.tool.function.FunctionToolCallback;
 import org.springframework.ai.tool.method.MethodToolCallback;
 import org.springframework.ai.tool.method.MethodToolCallbackProvider;
+import org.springframework.ai.tool.observation.ToolCallingContentObservationFilter;
 import org.springframework.ai.tool.resolution.DelegatingToolCallbackResolver;
 import org.springframework.ai.tool.resolution.ToolCallbackResolver;
 import org.springframework.ai.tool.support.ToolDefinitions;
@@ -111,6 +112,25 @@ void resolveMissingToolCallbacks() {
 			});
 	}
 
+	@Test
+	void observationFilterDefault() {
+		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(ToolCallingAutoConfiguration.class))
+			.withUserConfiguration(Config.class)
+			.run(context -> {
+				assertThat(context).doesNotHaveBean(ToolCallingContentObservationFilter.class);
+			});
+	}
+
+	@Test
+	void observationFilterEnabled() {
+		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(ToolCallingAutoConfiguration.class))
+			.withPropertyValues("spring.ai.tools.observations.include-content=true")
+			.withUserConfiguration(Config.class)
+			.run(context -> {
+				assertThat(context).hasSingleBean(ToolCallingContentObservationFilter.class);
+			});
+	}
+
 	static class WeatherService {
 
 		@Tool(description = "Get the weather in location. Return temperature in 36°F or 36°C format.")

spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java

@@ -67,6 +67,10 @@ public enum AiObservationAttributes {
 	 * The temperature setting for the model request.
 	 */
 	REQUEST_TEMPERATURE("gen_ai.request.temperature"),
+	/**
+	 * List of tool definitions provided to the model in the request.
+	 */
+	REQUEST_TOOL_NAMES("spring.ai.model.request.tool.names"),
 	/**
 	 * The top_k sampling setting for the model request.
 	 */

spring-ai-commons/src/main/java/org/springframework/ai/observation/conventions/SpringAiKind.java

@@ -37,6 +37,11 @@ public enum SpringAiKind {
 	 */
 	CHAT_CLIENT("chat_client"),
 
+	/**
+	 * Spring AI kind for tool calling.
+	 */
+	TOOL_CALL("tool_call"),
+
 	/**
 	 * Spring AI kind for vector store.
 	 */

spring-ai-docs/src/main/antora/modules/ROOT/pages/observability/index.adoc

@@ -155,6 +155,7 @@ IMPORTANT: The `gen_ai.client.token.usage` metrics measures number of input and
 |`gen_ai.usage.total_tokens` | The total number of tokens used in the model exchange.
 |`gen_ai.prompt` | The full prompt sent to the model. Optional.
 |`gen_ai.completion` | The full response received from the model. Optional.
+|`spring.ai.model.request.tool.names` | List of tool definitions provided to the model in the request.
 |===
 
 NOTE: For measuring user tokens, the previous table lists the values present in an observation trace.
@@ -179,6 +180,46 @@ Spring AI supports logging chat prompt and completion data, useful for troublesh
 
 WARNING: If you enable logging of the chat prompt and completion data, there's a risk of exposing sensitive or private information. Please, be careful!
 
+== Tool Calling
+
+The `spring.ai.tool` observations are recorded when performing tool calling in the context of a chat model interaction. They measure the time spent on toll call completion and propagate the related tracing information.
+
+.Low Cardinality Keys
+[cols="a,a", stripes=even]
+|===
+|Name | Description
+
+|`gen_ai.operation.name` | The name of the operation being performed. It's always `framework`.
+|`gen_ai.system` | The provider responsible for the operation. It's always `spring_ai`.
+|`spring.ai.kind` | The kind of operation performed by Spring AI. It's always `tool_call`.
+|`spring.ai.tool.definition.name` | The name of the tool.
+|===
+
+.High Cardinality Keys
+[cols="a,a", stripes=even]
+|===
+|Name | Description
+|`spring.ai.tool.definition.description` | Description of the tool.
+|`spring.ai.tool.definition.schema` | Schema of the parameters used to call the tool.
+|`spring.ai.tool.call.arguments` | The input arguments to the tool call. (Only when enabled)
+|`spring.ai.tool.call.result` | Schema of the parameters used to call the tool. (Only when enabled)
+|===
+
+=== Tool Call Arguments and Result Data
+
+The input arguments and result from the tool call are not exported by default, as they can be potentially sensitive.
+
+Spring AI supports exporting tool call arguments and result data as span attributes.
+
+[cols="6,3,1", stripes=even]
+|====
+| Property | Description | Default
+
+| `spring.ai.tools.observations.include-content` | Include the tool call content in observations. `true` or `false` | `false`
+|====
+
+WARNING: If you enable the inclusion of the tool call arguments and result in the observations, there's a risk of exposing sensitive or private information. Please, be careful!
+
 == EmbeddingModel
 
 NOTE: Observability features are currently supported only for `EmbeddingModel` implementations from the following

spring-ai-model/src/main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java

@@ -150,6 +150,16 @@ public String asString() {
 			}
 		},
 
+		/**
+		 * List of tool definitions provided to the model in the request.
+		 */
+		REQUEST_TOOL_NAMES {
+			@Override
+			public String asString() {
+				return AiObservationAttributes.REQUEST_TOOL_NAMES.value();
+			}
+		},
+
 		/**
 		 * The top_k sampling setting for the model request.
 		 */

spring-ai-model/src/main/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java

@@ -16,13 +16,15 @@
 
 package org.springframework.ai.chat.observation;
 
-import java.util.Objects;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.StringJoiner;
 
 import io.micrometer.common.KeyValue;
 import io.micrometer.common.KeyValues;
 
 import org.springframework.ai.chat.prompt.ChatOptions;
+import org.springframework.ai.model.tool.ToolCallingChatOptions;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 
@@ -100,6 +102,7 @@ public KeyValues getHighCardinalityKeyValues(ChatModelObservationContext context
 		keyValues = requestPresencePenalty(keyValues, context);
 		keyValues = requestStopSequences(keyValues, context);
 		keyValues = requestTemperature(keyValues, context);
+		keyValues = requestTools(keyValues, context);
 		keyValues = requestTopK(keyValues, context);
 		keyValues = requestTopP(keyValues, context);
 		// Response
@@ -148,8 +151,6 @@ protected KeyValues requestStopSequences(KeyValues keyValues, ChatModelObservati
 		if (!CollectionUtils.isEmpty(options.getStopSequences())) {
 			StringJoiner stopSequencesJoiner = new StringJoiner(", ", "[", "]");
 			options.getStopSequences().forEach(value -> stopSequencesJoiner.add("\"" + value + "\""));
-			KeyValue.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES,
-					options.getStopSequences(), Objects::nonNull);
 			return keyValues.and(
 					ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_STOP_SEQUENCES.asString(),
 					stopSequencesJoiner.toString());
@@ -167,6 +168,24 @@ protected KeyValues requestTemperature(KeyValues keyValues, ChatModelObservation
 		return keyValues;
 	}
 
+	protected KeyValues requestTools(KeyValues keyValues, ChatModelObservationContext context) {
+		if (!(context.getRequest().getOptions() instanceof ToolCallingChatOptions options)) {
+			return keyValues;
+		}
+
+		Set<String> toolNames = new HashSet<>(options.getToolNames());
+		toolNames.addAll(options.getToolCallbacks().stream().map(tc -> tc.getToolDefinition().name()).toList());
+
+		if (!CollectionUtils.isEmpty(toolNames)) {
+			StringJoiner toolNamesJoiner = new StringJoiner(", ", "[", "]");
+			toolNames.forEach(value -> toolNamesJoiner.add("\"" + value + "\""));
+			return keyValues.and(
+					ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_TOOL_NAMES.asString(),
+					toolNamesJoiner.toString());
+		}
+		return keyValues;
+	}
+
 	protected KeyValues requestTopK(KeyValues keyValues, ChatModelObservationContext context) {
 		ChatOptions options = context.getRequest().getOptions();
 		if (options.getTopK() != null) {