add agentkit_runtime_operation_latency metric for agentkit

Author: shunjiazhu

observability/attributes.go

@@ -17,12 +17,15 @@ package observability
 import (
 	"context"
 	"os"
+	"strings"
 
 	"github.com/volcengine/veadk-go/configs"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 )
 
+var isAgentKitRuntime = checkAgentKitRuntime()
+
 // setCommonAttributes enriches the span with common attributes from context, config, or env.
 func setCommonAttributes(ctx context.Context, span trace.Span) {
 	// 1. Fixed attributes
@@ -192,3 +195,21 @@ func getServiceName(cfg *configs.OpenTelemetryConfig) string {
 	}
 	return "<unknown_service>"
 }
+
+// OTEL_RESOURCE_ATTRIBUTES="instance.id=123456,apmplus.business_carrier=agentkit_runtime"
+func checkAgentKitRuntime() bool {
+	otelAttrStr := os.Getenv("OTEL_RESOURCE_ATTRIBUTES")
+	if otelAttrStr == "" {
+		return false
+	}
+
+	otelAttrs := strings.Split(otelAttrStr, ",")
+	for _, attr := range otelAttrs {
+		kv := strings.Split(attr, "=")
+		if len(kv) == 2 && kv[0] == "apmplus.business_carrier" && kv[1] == "agentkit_runtime" {
+			return true
+		}
+	}
+
+	return false
+}

observability/metrics.go

@@ -16,7 +16,6 @@ package observability
 
 import (
 	"context"
-	"fmt"
 	"sync"
 
 	"go.opentelemetry.io/otel"
@@ -290,10 +289,7 @@ func RecordAPMPlusToolTokenUsage(ctx context.Context, tokens int64, attrs ...att
 	}
 }
 
-func RecordAgentKitDuration(ctx context.Context, durationSeconds float64, err error, attrs ...attribute.KeyValue) {
-	if err != nil {
-		attrs = append(attrs, attribute.String("error_type", fmt.Sprintf("%T", err)))
-	}
+func RecordAgentKitDuration(ctx context.Context, durationSeconds float64, attrs ...attribute.KeyValue) {
 	for _, histogram := range agentkitDurationHistograms {
 		histogram.Record(ctx, durationSeconds, metric.WithAttributes(attrs...))
 	}

observability/metrics_test.go

@@ -16,7 +16,6 @@ package observability
 
 import (
 	"context"
-	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -120,8 +119,7 @@ func TestMetricsRecording(t *testing.T) {
 	})
 
 	t.Run("RecordAgentKitDurationWithError", func(t *testing.T) {
-		testErr := fmt.Errorf("test error")
-		RecordAgentKitDuration(ctx, 2.5, testErr, attrs...)
+		RecordAgentKitDuration(ctx, 2.5, attrs...)
 
 		var rm metricdata.ResourceMetrics
 		err := reader.Collect(ctx, &rm)
@@ -136,8 +134,6 @@ func TestMetricsRecording(t *testing.T) {
 						if dp.Count > 0 {
 							assert.Equal(t, uint64(1), dp.Count)
 							assert.Equal(t, 2.5, dp.Sum)
-							errType, _ := dp.Attributes.Value("error_type")
-							assert.Equal(t, "*errors.errorString", errType.AsString())
 							found = true
 						}
 					}

observability/plugin.go

@@ -189,16 +189,22 @@ func (p *adkObservabilityPlugin) AfterRun(ctx agent.InvocationContext) {
 					RecordOperationDuration(context.Background(), elapsed, metricAttrs...)
 					RecordAPMPlusSpanLatency(context.Background(), elapsed, metricAttrs...)
 
-					agentKitsAttrs := []attribute.KeyValue{
-						attribute.String("gen_ai_operation_name", "chain"),
-						attribute.String("gen_ai_operation_type", "workflow"),
+					if isAgentKitRuntime {
+						agentKitsAttrs := []attribute.KeyValue{
+							attribute.String("gen_ai_operation_name", "chain"),
+							attribute.String("gen_ai_operation_type", "workflow"),
+						}
+
+						var lastErr error
+						if val, _ := ctx.Session().State().Get(stateKeyLastError); val != nil {
+							lastErr = val.(error)
+						}
+						if lastErr != nil {
+							agentKitsAttrs = append(agentKitsAttrs, attribute.String("error_type", "invocation_error"))
+						}
+						RecordAgentKitDuration(context.Background(), elapsed, agentKitsAttrs...)
 					}
 
-					var lastErr error
-					if val, _ := ctx.Session().State().Get(stateKeyLastError); val != nil {
-						lastErr = val.(error)
-					}
-					RecordAgentKitDuration(context.Background(), elapsed, lastErr, agentKitsAttrs...)
 				}
 
 			}
@@ -367,7 +373,7 @@ func (p *adkObservabilityPlugin) AfterModel(ctx agent.CallbackContext, resp *mod
 				attribute.String("gen_ai_response_model", meta.ModelName),
 				attribute.String("gen_ai_operation_name", "chat"),
 				attribute.String("gen_ai_operation_type", "llm"),
-				attribute.String("error_type", "error"), // Simple error type
+				attribute.String("error_type", "llm_error"), // Simple error type
 			}
 			RecordExceptions(context.Context(ctx), 1, metricAttrs...)
 			p.recordFinalResponseMetrics(ctx, meta, meta.ModelName, err)
@@ -590,11 +596,16 @@ func (p *adkObservabilityPlugin) recordFinalResponseMetrics(ctx agent.CallbackCo
 			RecordOperationDuration(context.Context(ctx), duration, metricAttrs...)
 			RecordAPMPlusSpanLatency(context.Context(ctx), duration, metricAttrs...)
 
-			agentKitsAttrs := []attribute.KeyValue{
-				attribute.String("gen_ai_operation_name", "chat"),
-				attribute.String("gen_ai_operation_type", "llm"),
+			if isAgentKitRuntime {
+				agentKitsAttrs := []attribute.KeyValue{
+					attribute.String("gen_ai_operation_name", "chat"),
+					attribute.String("gen_ai_operation_type", "llm"),
+				}
+				if err != nil {
+					agentKitsAttrs = append(agentKitsAttrs, attribute.String("error_type", "llm_error"))
+				}
+				RecordAgentKitDuration(context.Context(ctx), duration, agentKitsAttrs...)
 			}
-			RecordAgentKitDuration(context.Context(ctx), duration, err, agentKitsAttrs...)
 		}
 	}
 }
@@ -937,11 +948,16 @@ func (p *adkObservabilityPlugin) AfterTool(ctx tool.Context, tool tool.Tool, arg
 			RecordOperationDuration(context.Background(), duration, metricAttrs...)
 			RecordAPMPlusSpanLatency(context.Background(), duration, metricAttrs...)
 
-			agentKitsAttrs := []attribute.KeyValue{
-				attribute.String("gen_ai_operation_name", tool.Name()),
-				attribute.String("gen_ai_operation_type", "tool"),
+			if isAgentKitRuntime {
+				agentKitsAttrs := []attribute.KeyValue{
+					attribute.String("gen_ai_operation_name", tool.Name()),
+					attribute.String("gen_ai_operation_type", "tool"),
+				}
+				if err == nil {
+					agentKitsAttrs = append(agentKitsAttrs, attribute.String("error_type", "tool_error"))
+				}
+				RecordAgentKitDuration(context.Background(), duration, agentKitsAttrs...)
 			}
-			RecordAgentKitDuration(context.Background(), duration, err, agentKitsAttrs...)
 		}
 
 		if p.isMetricsEnabled() {
@@ -1034,6 +1050,28 @@ func (p *adkObservabilityPlugin) AfterAgent(ctx agent.CallbackContext) (*genai.C
 			span.End()
 		}
 	}
+
+	meta := p.getSpanMetadata(ctx.State())
+	if !meta.StartTime.IsZero() {
+		if p.isMetricsEnabled() {
+			if isAgentKitRuntime {
+				duration := time.Since(meta.StartTime).Seconds()
+				agentKitsAttrs := []attribute.KeyValue{
+					attribute.String("gen_ai_operation_name", "invoke_agent"),
+					attribute.String("gen_ai_operation_type", "agent_server"),
+				}
+				var lastErr error
+				if val, _ := ctx.State().Get(stateKeyLastError); val != nil {
+					lastErr = val.(error)
+				}
+
+				if lastErr != nil {
+					agentKitsAttrs = append(agentKitsAttrs, attribute.String("error_type", "invoke_agent_error"))
+				}
+				RecordAgentKitDuration(context.Background(), duration, agentKitsAttrs...)
+			}
+		}
+	}
 	return nil, nil
 }