Skip to content

debug logs #9389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gary-huang wants to merge 2 commits into from
Closed

debug logs #9389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
889f67d
debug logs
gary-huang Aug 19, 2025
dfdab0c
add more debug logs on span creation
gary-huang Aug 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/LLMObsSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,15 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) {

sco.createRemaining(config);

LOGGER.debug("LLM OBS has been created, setting span factory");
LLMObsInternal.setLLMObsSpanFactory(
new LLMObsManualSpanFactory(config.getLlmObsMlApp(), config.getServiceName()));

String mlApp = config.getLlmObsMlApp();
LLMObsInternal.setLLMObsSpanFactory(
new LLMObsManualSpanFactory(mlApp, config.getServiceName()));
LLMObs.LLMObsSpanFactory factory = new LLMObsManualSpanFactory(mlApp, config.getServiceName());
LLMObsInternal.setLLMObsSpanFactory(factory);

LOGGER.debug("LLM OBS SPAN FACTORY SET {}", factory);

LLMObsInternal.setLLMObsEvalProcessor(new LLMObsCustomEvalProcessor(mlApp, sco, config));
}
Expand Down Expand Up @@ -145,6 +148,8 @@ public LLMObsSpan startLLMSpan(
@Nullable String mlApp,
@Nullable String sessionId) {

LOGGER.debug("LLM OBS STARTING LLM SPAN");

DDLLMObsSpan span =
new DDLLMObsSpan(
Tags.LLMOBS_LLM_SPAN_KIND, spanName, getMLApp(mlApp), sessionId, serviceName);
Expand All @@ -158,6 +163,8 @@ public LLMObsSpan startLLMSpan(
modelProvider = CUSTOM_MODEL_VAL;
}
span.setTag(LLMObsTags.MODEL_PROVIDER, modelProvider);

LOGGER.debug("LLM OBS STARTED LLM SPAN {}", span);
return span;
}

Expand Down
1 change: 1 addition & 0 deletions dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ public void addThrowable(Throwable throwable) {

@Override
public void finish() {
LOGGER.debug("FINISHING LLM OBS SPAN {}", this);
if (finished) {
return;
}
Expand Down
4 changes: 4 additions & 0 deletions dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class LLMObs {
protected LLMObs() {}

protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE;
protected static LLMObsEvalProcessor EVAL_PROCESSOR = NoOpLLMObsEvalProcessor.INSTANCE;
private static final Logger LOGGER = LoggerFactory.getLogger(NoOpLLMObsSpanFactory.class);

public static LLMObsSpan startLLMSpan(
String spanName,
Expand All @@ -19,6 +22,7 @@ public static LLMObsSpan startLLMSpan(
@Nullable String mlApp,
@Nullable String sessionId) {

LOGGER.debug("LLM OBS SPAN FACTORY {}", SPAN_FACTORY);
return SPAN_FACTORY.startLLMSpan(spanName, modelName, modelProvider, mlApp, sessionId);
}

Expand Down
7 changes: 7 additions & 0 deletions dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpanFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,23 @@
import datadog.trace.api.llmobs.LLMObs;
import datadog.trace.api.llmobs.LLMObsSpan;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class NoOpLLMObsSpanFactory implements LLMObs.LLMObsSpanFactory {
public static final NoOpLLMObsSpanFactory INSTANCE = new NoOpLLMObsSpanFactory();

private static final Logger LOGGER = LoggerFactory.getLogger(NoOpLLMObsSpanFactory.class);

public LLMObsSpan startLLMSpan(
String spanName,
String modelName,
String modelProvider,
@Nullable String mlApp,
@Nullable String sessionId) {

LOGGER.debug("LLM OBS STARTED NOOP LLM SPAN");

return NoOpLLMObsSpan.INSTANCE;
}

Expand Down
14 changes: 13 additions & 1 deletion dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@ public void map(List<? extends CoreSpan<?>> trace, Writable writable) {
List<? extends CoreSpan<?>> llmobsSpans =
trace.stream().filter(LLMObsSpanMapper::isLLMObsSpan).collect(Collectors.toList());

if (llmobsSpans.size() <= 0) {
LOGGER.debug("no LLMObs spans found, {} spans found before filtering", trace.size());
return;
}

writable.startMap(3);

writable.writeUTF8(EVENT_TYPE);
Expand Down Expand Up @@ -147,7 +152,10 @@ public void map(List<? extends CoreSpan<?>> trace, Writable writable) {

private static boolean isLLMObsSpan(CoreSpan<?> span) {
CharSequence type = span.getType();
return type != null && type.toString().contentEquals(InternalSpanTypes.LLMOBS);

boolean filtered = type != null && type.toString().contentEquals(InternalSpanTypes.LLMOBS);
LOGGER.debug("INCLUDED={} span {}", filtered, span);
return filtered;
}

@Override
Expand Down Expand Up @@ -214,6 +222,7 @@ public void accept(Metadata metadata) {
String spanKind = "unknown";
for (Map.Entry<String, Object> tag : metadata.getTags().entrySet()) {
String key = tag.getKey();
LOGGER.debug("LLM OBS SPAN TAG: key={} val={}", key, tag.getValue());
if (key.equals(SPAN_KIND_TAG_KEY)) {
spanKind = String.valueOf(tag.getValue());
} else if (TAGS_FOR_REMAPPING.contains(key)) {
Expand Down Expand Up @@ -243,6 +252,7 @@ public void accept(Metadata metadata) {
writable.startMap(metricsSize);
for (Map.Entry<String, Object> tag : metadata.getTags().entrySet()) {
String tagKey = tag.getKey();
LOGGER.debug("LLM OBS FINAL METRIC: key={} val={}", tagKey, tag.getValue());
if (tagKey.startsWith(LLMOBS_METRIC_PREFIX) && tag.getValue() instanceof Number) {
writable.writeString(tagKey.substring(LLMOBS_METRIC_PREFIX.length()), null);
writable.writeObject(tag.getValue(), null);
Expand All @@ -256,6 +266,7 @@ public void accept(Metadata metadata) {
for (Map.Entry<String, Object> tag : metadata.getTags().entrySet()) {
String key = tag.getKey();
Object value = tag.getValue();
LOGGER.debug("LLM OBS FINAL TAG: key={} val={}", key, value);
if (!tagsToRemapToMeta.containsKey(key) && key.startsWith(LLMOBS_TAG_PREFIX)) {
writable.writeObject(key.substring(LLMOBS_TAG_PREFIX.length()) + ":" + value, null);
}
Expand All @@ -276,6 +287,7 @@ public void accept(Metadata metadata) {
for (Map.Entry<String, Object> tag : tagsToRemapToMeta.entrySet()) {
String key = tag.getKey().substring(LLMOBS_TAG_PREFIX.length());
Object val = tag.getValue();
LOGGER.debug("LLM OBS REMAPPED META SPAN TAG: key={} val={}", key, val);
if (key.equals(INPUT) || key.equals(OUTPUT)) {
if (!spanKind.equals(Tags.LLMOBS_LLM_SPAN_KIND)) {
key += ".value";
Expand Down