Skip to content

Commit 5fc9069

Browse files
gary-huanggary-huang
committed
add api for evals
1 parent f78b2c5 commit 5fc9069

File tree

7 files changed

+123
-5
lines changed

7 files changed

+123
-5
lines changed

‎dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import datadog.context.ContextScope;
44
import datadog.trace.api.DDSpanTypes;
5+
import datadog.trace.api.DDTraceId;
56
import datadog.trace.api.llmobs.LLMObs;
67
import datadog.trace.api.llmobs.LLMObsSpan;
78
import datadog.trace.api.llmobs.LLMObsTags;
@@ -67,18 +68,22 @@ public DDLLMObsSpan(
6768
if (sessionID != null && !sessionID.isEmpty()) {
6869
this.span.setTag(LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID, sessionID);
6970
}
70-
71+
7172
AgentSpanContext parent = LLMObsState.getLLMObsParentContext();
7273
String parentSpanID = LLMObsState.ROOT_SPAN_ID;
7374
if (null != parent) {
7475
if (parent.getTraceId() != this.span.getTraceId()) {
75-
LOGGER.error("trace ID mismatch, retrieved parent from context trace_id={}, span_id={}, started span trace_id={}, span_id={}", parent.getTraceId(), parent.getSpanId(), this.span.getTraceId(), this.span.getSpanId());
76+
LOGGER.error(
77+
"trace ID mismatch, retrieved parent from context trace_id={}, span_id={}, started span trace_id={}, span_id={}",
78+
parent.getTraceId(),
79+
parent.getSpanId(),
80+
this.span.getTraceId(),
81+
this.span.getSpanId());
7682
} else {
7783
parentSpanID = String.valueOf(parent.getSpanId());
7884
}
7985
}
80-
this.span.setTag(
81-
LLMOBS_TAG_PREFIX + PARENT_ID_TAG_INTERNAL, parentSpanID);
86+
this.span.setTag(LLMOBS_TAG_PREFIX + PARENT_ID_TAG_INTERNAL, parentSpanID);
8287
this.scope = LLMObsState.attach();
8388
LLMObsState.setLLMObsParentContext(this.span.context());
8489
}
@@ -292,4 +297,14 @@ public void finish() {
292297
this.scope.close();
293298
this.finished = true;
294299
}
300+
301+
@Override
302+
public DDTraceId getTraceId() {
303+
return this.span.getTraceId();
304+
}
305+
306+
@Override
307+
public long getSpanId() {
308+
return this.span.getSpanId();
309+
}
295310
}

‎dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/LLMObsState.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,4 @@ public static void setLLMObsParentContext(AgentSpanContext llmObsParentContext)
3434
state.parentSpanID = llmObsParentContext;
3535
}
3636
}
37-
3837
}

‎dd-trace-api/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ excludedClassesCoverage += [
3838
'datadog.trace.api.llmobs.LLMObsSpan',
3939
'datadog.trace.api.llmobs.noop.NoOpLLMObsSpan',
4040
'datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory',
41+
'datadog.trace.api.llmobs.noop.NoOpLLMObsEvalProcessor',
4142
'datadog.trace.api.experimental.DataStreamsCheckpointer',
4243
'datadog.trace.api.experimental.DataStreamsCheckpointer.NoOp',
4344
'datadog.trace.api.experimental.DataStreamsContextCarrier',

‎dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package datadog.trace.api.llmobs;
22

3+
import datadog.trace.api.llmobs.noop.NoOpLLMObsEvalProcessor;
34
import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory;
45
import java.util.List;
56
import java.util.Map;
@@ -9,6 +10,7 @@ public class LLMObs {
910
protected LLMObs() {}
1011

1112
protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE;
13+
protected static LLMObsEvalProcessor EVAL_PROCESSOR = NoOpLLMObsEvalProcessor.INSTANCE;
1214

1315
public static LLMObsSpan startLLMSpan(
1416
String spanName,
@@ -44,6 +46,26 @@ public static LLMObsSpan startWorkflowSpan(
4446
return SPAN_FACTORY.startWorkflowSpan(spanName, mlApp, sessionID);
4547
}
4648

49+
public static void SubmitEvaluation(
50+
LLMObsSpan llmObsSpan, String label, String categoricalValue, Map<String, Object> tags) {
51+
EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, categoricalValue, tags);
52+
}
53+
54+
public static void SubmitEvaluation(
55+
LLMObsSpan llmObsSpan, String label, String categoricalValue, String mlApp, Map<String, Object> tags) {
56+
EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, categoricalValue, mlApp, tags);
57+
}
58+
59+
public static void SubmitEvaluation(
60+
LLMObsSpan llmObsSpan, String label, double scoreValue, Map<String, Object> tags) {
61+
EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, scoreValue, tags);
62+
}
63+
64+
public static void SubmitEvaluation(
65+
LLMObsSpan llmObsSpan, String label, double scoreValue, String mlApp, Map<String, Object> tags) {
66+
EVAL_PROCESSOR.SubmitEvaluation(llmObsSpan, label, scoreValue, mlApp, tags);
67+
}
68+
4769
public interface LLMObsSpanFactory {
4870
LLMObsSpan startLLMSpan(
4971
String spanName,
@@ -62,6 +84,28 @@ LLMObsSpan startWorkflowSpan(
6284
String spanName, @Nullable String mlApp, @Nullable String sessionID);
6385
}
6486

87+
public interface LLMObsEvalProcessor {
88+
void SubmitEvaluation(
89+
LLMObsSpan llmObsSpan, String label, double scoreValue, Map<String, Object> tags);
90+
91+
void SubmitEvaluation(
92+
LLMObsSpan llmObsSpan,
93+
String label,
94+
double scoreValue,
95+
String mlApp,
96+
Map<String, Object> tags);
97+
98+
void SubmitEvaluation(
99+
LLMObsSpan llmObsSpan, String label, String categoricalValue, Map<String, Object> tags);
100+
101+
void SubmitEvaluation(
102+
LLMObsSpan llmObsSpan,
103+
String label,
104+
String categoricalValue,
105+
String mlApp,
106+
Map<String, Object> tags);
107+
}
108+
65109
public static class ToolCall {
66110
private String name;
67111
private String type;

‎dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package datadog.trace.api.llmobs;
22

3+
import datadog.trace.api.DDTraceId;
34
import java.util.List;
45
import java.util.Map;
56

@@ -142,4 +143,18 @@ public interface LLMObsSpan {
142143

143144
/** Finishes (closes) a span */
144145
void finish();
146+
147+
/**
148+
* Gets the TraceId of the span's trace.
149+
*
150+
* @return The TraceId of the span's trace, or {@link DDTraceId#ZERO} if not set.
151+
*/
152+
DDTraceId getTraceId();
153+
154+
/**
155+
* Gets the SpanId.
156+
*
157+
* @return The span identifier.
158+
*/
159+
long getSpanId();
145160
}

‎dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsEvalProcessor.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package datadog.trace.api.llmobs.noop;
2+
3+
import datadog.trace.api.llmobs.LLMObs;
4+
import datadog.trace.api.llmobs.LLMObsSpan;
5+
import java.util.Map;
6+
7+
public class NoOpLLMObsEvalProcessor implements LLMObs.LLMObsEvalProcessor {
8+
public static final NoOpLLMObsEvalProcessor INSTANCE = new NoOpLLMObsEvalProcessor();
9+
10+
@Override
11+
public void SubmitEvaluation(
12+
LLMObsSpan llmObsSpan, String label, double scoreValue, Map<String, Object> tags) {}
13+
14+
@Override
15+
public void SubmitEvaluation(
16+
LLMObsSpan llmObsSpan,
17+
String label,
18+
double scoreValue,
19+
String mlApp,
20+
Map<String, Object> tags) {}
21+
22+
@Override
23+
public void SubmitEvaluation(
24+
LLMObsSpan llmObsSpan, String label, String categoricalValue, Map<String, Object> tags) {}
25+
26+
@Override
27+
public void SubmitEvaluation(
28+
LLMObsSpan llmObsSpan,
29+
String label,
30+
String categoricalValue,
31+
String mlApp,
32+
Map<String, Object> tags) {}
33+
}

‎dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package datadog.trace.api.llmobs.noop;
22

3+
import datadog.trace.api.DDTraceId;
34
import datadog.trace.api.llmobs.LLMObs;
45
import datadog.trace.api.llmobs.LLMObsSpan;
56
import java.util.List;
@@ -58,4 +59,14 @@ public void addThrowable(Throwable throwable) {}
5859

5960
@Override
6061
public void finish() {}
62+
63+
@Override
64+
public DDTraceId getTraceId() {
65+
return DDTraceId.ZERO;
66+
}
67+
68+
@Override
69+
public long getSpanId() {
70+
return 0;
71+
}
6172
}

0 commit comments

Comments
 (0)