diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java index 00a04e5e59d..1a449d8f97c 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java @@ -31,12 +31,15 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { sco.createRemaining(config); + LOGGER.debug("LLM OBS has been created, setting span factory"); LLMObsInternal.setLLMObsSpanFactory( new LLMObsManualSpanFactory(config.getLlmObsMlApp(), config.getServiceName())); String mlApp = config.getLlmObsMlApp(); - LLMObsInternal.setLLMObsSpanFactory( - new LLMObsManualSpanFactory(mlApp, config.getServiceName())); + LLMObs.LLMObsSpanFactory factory = new LLMObsManualSpanFactory(mlApp, config.getServiceName()); + LLMObsInternal.setLLMObsSpanFactory(factory); + + LOGGER.debug("LLM OBS SPAN FACTORY SET {}", factory); LLMObsInternal.setLLMObsEvalProcessor(new LLMObsCustomEvalProcessor(mlApp, sco, config)); } @@ -145,6 +148,8 @@ public LLMObsSpan startLLMSpan( @Nullable String mlApp, @Nullable String sessionId) { + LOGGER.debug("LLM OBS STARTING LLM SPAN"); + DDLLMObsSpan span = new DDLLMObsSpan( Tags.LLMOBS_LLM_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName); @@ -158,6 +163,8 @@ public LLMObsSpan startLLMSpan( modelProvider = CUSTOM_MODEL_VAL; } span.setTag(LLMObsTags.MODEL_PROVIDER, modelProvider); + + LOGGER.debug("LLM OBS STARTED LLM SPAN {}", span); return span; } diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 71d13596090..4692fad6ad6 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -290,6 +290,7 @@ public void addThrowable(Throwable throwable) { @Override public void finish() { + LOGGER.debug("FINISHING LLM OBS SPAN {}", this); if (finished) { return; } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 6f3224dfcaa..00b5ce2ed8a 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -5,12 +5,15 @@ import java.util.List; import java.util.Map; import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LLMObs { protected LLMObs() {} protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE; protected static LLMObsEvalProcessor EVAL_PROCESSOR = NoOpLLMObsEvalProcessor.INSTANCE; + private static final Logger LOGGER = LoggerFactory.getLogger(NoOpLLMObsSpanFactory.class); public static LLMObsSpan startLLMSpan( String spanName, @@ -19,6 +22,7 @@ public static LLMObsSpan startLLMSpan( @Nullable String mlApp, @Nullable String sessionId) { + LOGGER.debug("LLM OBS SPAN FACTORY {}", SPAN_FACTORY); return SPAN_FACTORY.startLLMSpan(spanName, modelName, modelProvider, mlApp, sessionId); } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java index b142ee43313..baebbf49ffb 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java @@ -3,16 +3,23 @@ import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsSpan; import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class NoOpLLMObsSpanFactory implements LLMObs.LLMObsSpanFactory { public static final NoOpLLMObsSpanFactory INSTANCE = new NoOpLLMObsSpanFactory(); + private static final Logger LOGGER = LoggerFactory.getLogger(NoOpLLMObsSpanFactory.class); + public LLMObsSpan startLLMSpan( String spanName, String modelName, String modelProvider, @Nullable String mlApp, @Nullable String sessionId) { + + LOGGER.debug("LLM OBS STARTED NOOP LLM SPAN"); + return NoOpLLMObsSpan.INSTANCE; } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index fce921787e2..0439cd97bba 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -93,6 +93,11 @@ public void map(List> trace, Writable writable) { List> llmobsSpans = trace.stream().filter(LLMObsSpanMapper::isLLMObsSpan).collect(Collectors.toList()); + if (llmobsSpans.size() <= 0) { + LOGGER.debug("no LLMObs spans found, {} spans found before filtering", trace.size()); + return; + } + writable.startMap(3); writable.writeUTF8(EVENT_TYPE); @@ -147,7 +152,10 @@ public void map(List> trace, Writable writable) { private static boolean isLLMObsSpan(CoreSpan span) { CharSequence type = span.getType(); - return type != null && type.toString().contentEquals(InternalSpanTypes.LLMOBS); + + boolean filtered = type != null && type.toString().contentEquals(InternalSpanTypes.LLMOBS); + LOGGER.debug("INCLUDED={} span {}", filtered, span); + return filtered; } @Override @@ -214,6 +222,7 @@ public void accept(Metadata metadata) { String spanKind = "unknown"; for (Map.Entry tag : metadata.getTags().entrySet()) { String key = tag.getKey(); + LOGGER.debug("LLM OBS SPAN TAG: key={} val={}", key, tag.getValue()); if (key.equals(SPAN_KIND_TAG_KEY)) { spanKind = String.valueOf(tag.getValue()); } else if (TAGS_FOR_REMAPPING.contains(key)) { @@ -243,6 +252,7 @@ public void accept(Metadata metadata) { writable.startMap(metricsSize); for (Map.Entry tag : metadata.getTags().entrySet()) { String tagKey = tag.getKey(); + LOGGER.debug("LLM OBS FINAL METRIC: key={} val={}", tagKey, tag.getValue()); if (tagKey.startsWith(LLMOBS_METRIC_PREFIX) && tag.getValue() instanceof Number) { writable.writeString(tagKey.substring(LLMOBS_METRIC_PREFIX.length()), null); writable.writeObject(tag.getValue(), null); @@ -256,6 +266,7 @@ public void accept(Metadata metadata) { for (Map.Entry tag : metadata.getTags().entrySet()) { String key = tag.getKey(); Object value = tag.getValue(); + LOGGER.debug("LLM OBS FINAL TAG: key={} val={}", key, value); if (!tagsToRemapToMeta.containsKey(key) && key.startsWith(LLMOBS_TAG_PREFIX)) { writable.writeObject(key.substring(LLMOBS_TAG_PREFIX.length()) + ":" + value, null); } @@ -276,6 +287,7 @@ public void accept(Metadata metadata) { for (Map.Entry tag : tagsToRemapToMeta.entrySet()) { String key = tag.getKey().substring(LLMOBS_TAG_PREFIX.length()); Object val = tag.getValue(); + LOGGER.debug("LLM OBS REMAPPED META SPAN TAG: key={} val={}", key, val); if (key.equals(INPUT) || key.equals(OUTPUT)) { if (!spanKind.equals(Tags.LLMOBS_LLM_SPAN_KIND)) { key += ".value";