Skip to content

Commit 972b2ac

Browse files
ygreeygree
committed
test_responses_create_tool_input WIP
1 parent 4282f65 commit 972b2ac

File tree

7 files changed

+571
-26
lines changed

7 files changed

+571
-26
lines changed

‎dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
11
apply from: "$rootDir/gradle/java.gradle"
22
apply plugin: 'idea'
33

4-
// ChatCompletionMessageFunctionToolCall introduced in v3.0.0
54
def minVer = '3.0.0'
65

76
muzzle {
87
pass {
98
group = "com.openai"
109
module = "openai-java"
1110
versions = "[$minVer,)"
12-
// assertInverse = true //TODO fix after module split
1311
}
1412
}
1513

‎dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/FunctionCallOutputExtractor.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package datadog.trace.instrumentation.openai_java;
2+
3+
import com.openai.models.responses.ResponseInputItem;
4+
import datadog.trace.util.MethodHandles;
5+
import java.lang.invoke.MethodHandle;
6+
import org.slf4j.Logger;
7+
import org.slf4j.LoggerFactory;
8+
9+
/**
10+
* Helper class to handle FunctionCallOutput.output() method changes between openai-java versions.
11+
*
12+
* <p>In version 3.x: output() returns String In version 4.0+: output() returns Output)
13+
*/
14+
public class FunctionCallOutputExtractor {
15+
private static final Logger log = LoggerFactory.getLogger(FunctionCallOutputExtractor.class);
16+
17+
private static final MethodHandles METHOD_HANDLES =
18+
new MethodHandles(ResponseInputItem.FunctionCallOutput.class.getClassLoader());
19+
20+
private static final MethodHandle OUTPUT_METHOD;
21+
private static final MethodHandle IS_STRING_METHOD;
22+
private static final MethodHandle AS_STRING_METHOD;
23+
24+
static {
25+
OUTPUT_METHOD =
26+
METHOD_HANDLES.method(ResponseInputItem.FunctionCallOutput.class, "output");
27+
28+
Class<?> outputClass = null;
29+
try {
30+
outputClass =
31+
ResponseInputItem.FunctionCallOutput.class
32+
.getClassLoader()
33+
.loadClass("com.openai.models.responses.ResponseInputItem$FunctionCallOutput$Output");
34+
} catch (ClassNotFoundException e) {
35+
// Output class not found, assuming openai-java version 3.x
36+
}
37+
38+
if (outputClass != null) {
39+
IS_STRING_METHOD = METHOD_HANDLES.method(outputClass, "isString");
40+
AS_STRING_METHOD = METHOD_HANDLES.method(outputClass, "asString");
41+
} else {
42+
IS_STRING_METHOD = null;
43+
AS_STRING_METHOD = null;
44+
}
45+
}
46+
47+
public static String getOutputAsString(ResponseInputItem.FunctionCallOutput functionCallOutput) {
48+
try {
49+
Object output = METHOD_HANDLES.invoke(OUTPUT_METHOD, functionCallOutput);
50+
51+
if (output == null) {
52+
return null;
53+
}
54+
55+
// In v3.x, output() returns String directly
56+
if (output instanceof String) {
57+
return (String) output;
58+
}
59+
60+
// In v4.0+, output() returns an Output object
61+
if (IS_STRING_METHOD != null && AS_STRING_METHOD != null) {
62+
Boolean isString = METHOD_HANDLES.invoke(IS_STRING_METHOD, output);
63+
if (Boolean.TRUE.equals(isString)) {
64+
return METHOD_HANDLES.invoke(AS_STRING_METHOD, output);
65+
} else {
66+
log.debug(
67+
"FunctionCallOutput.output() returned non-string Output type, skipping");
68+
return null;
69+
}
70+
}
71+
72+
log.debug(
73+
"Unable to extract string from FunctionCallOutput.output(): unexpected return type {}",
74+
output.getClass().getName());
75+
return null;
76+
77+
} catch (Exception e) {
78+
log.debug("Error extracting output from FunctionCallOutput", e);
79+
return null;
80+
}
81+
}
82+
}
83+

‎dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java‎

Lines changed: 260 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import com.openai.models.responses.Response;
1010
import com.openai.models.responses.ResponseCreateParams;
1111
import com.openai.models.responses.ResponseFunctionToolCall;
12+
import com.openai.models.responses.ResponseInputContent;
13+
import com.openai.models.responses.ResponseInputItem;
1214
import com.openai.models.responses.ResponseOutputItem;
1315
import com.openai.models.responses.ResponseOutputMessage;
1416
import com.openai.models.responses.ResponseOutputText;
@@ -54,11 +56,49 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params
5456
inputMessages.add(LLMObs.LLMMessage.from("system", instructions));
5557
});
5658

57-
Optional<String> textOpt = params._input().asString();
59+
Optional<String> textOpt = params._input().asString(); // TODO cover with unit tests
5860
if (textOpt.isPresent()) {
5961
inputMessages.add(LLMObs.LLMMessage.from("user", textOpt.get()));
6062
}
6163

64+
Optional<ResponseCreateParams.Input> inputOpt = params._input().asKnown();
65+
if (inputOpt.isPresent()) {
66+
ResponseCreateParams.Input input = inputOpt.get();
67+
if (input.isText()) {
68+
inputMessages.add(LLMObs.LLMMessage.from("user", input.asText()));
69+
} else if (input.isResponse()) {
70+
List<ResponseInputItem> inputItems = input.asResponse(); // TODO cover with unit tests
71+
for (ResponseInputItem item : inputItems) {
72+
LLMObs.LLMMessage message = extractInputItemMessage(item);
73+
if (message != null) {
74+
inputMessages.add(message);
75+
}
76+
}
77+
}
78+
}
79+
80+
// Handle raw list input (when SDK can't parse into known types)
81+
// This path is tested by "create streaming response with raw json tool input test"
82+
if (inputMessages.isEmpty()) {
83+
try {
84+
Optional<com.openai.core.JsonValue> rawValueOpt = params._input().asUnknown();
85+
if (rawValueOpt.isPresent()) {
86+
com.openai.core.JsonValue rawValue = rawValueOpt.get();
87+
Optional<List<com.openai.core.JsonValue>> rawListOpt = rawValue.asArray();
88+
if (rawListOpt.isPresent()) {
89+
for (com.openai.core.JsonValue item : rawListOpt.get()) {
90+
LLMObs.LLMMessage message = extractMessageFromRawJson(item);
91+
if (message != null) {
92+
inputMessages.add(message);
93+
}
94+
}
95+
}
96+
}
97+
} catch (Exception e) {
98+
// Ignore parsing errors for raw input
99+
}
100+
}
101+
62102
if (!inputMessages.isEmpty()) {
63103
span.setTag("_ml_obs_tag.input", inputMessages);
64104
}
@@ -67,6 +107,225 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params
67107
.ifPresent(reasoningMap -> span.setTag("_ml_obs_request.reasoning", reasoningMap));
68108
}
69109

110+
private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) {
111+
if (item.isMessage()) {
112+
ResponseInputItem.Message message = item.asMessage();
113+
String role = message.role().asString();
114+
String content = extractInputMessageContent(message);
115+
return LLMObs.LLMMessage.from(role, content);
116+
} else if (item.isFunctionCall()) {
117+
// Function call is mapped to assistant message with tool_calls
118+
ResponseFunctionToolCall functionCall = item.asFunctionCall();
119+
LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(functionCall);
120+
if (toolCall != null) {
121+
List<LLMObs.ToolCall> toolCalls = Collections.singletonList(toolCall);
122+
return LLMObs.LLMMessage.from("assistant", null, toolCalls);
123+
}
124+
} else if (item.isFunctionCallOutput()) {
125+
ResponseInputItem.FunctionCallOutput output = item.asFunctionCallOutput();
126+
String callId = output.callId();
127+
String result = FunctionCallOutputExtractor.getOutputAsString(output);
128+
LLMObs.ToolResult toolResult =
129+
LLMObs.ToolResult.from("", "function_call_output", callId, result);
130+
List<LLMObs.ToolResult> toolResults = Collections.singletonList(toolResult);
131+
return LLMObs.LLMMessage.fromToolResults("user", toolResults);
132+
}
133+
return null;
134+
}
135+
136+
private LLMObs.LLMMessage extractMessageFromRawJson(com.openai.core.JsonValue jsonValue) {
137+
Optional<Map<String, com.openai.core.JsonValue>> objOpt = jsonValue.asObject();
138+
if (!objOpt.isPresent()) {
139+
return null;
140+
}
141+
142+
Map<String, com.openai.core.JsonValue> obj = objOpt.get();
143+
com.openai.core.JsonValue typeValue = obj.get("type");
144+
145+
// Check if it's a function_call
146+
if (typeValue != null) {
147+
Optional<String> typeStr = typeValue.asString();
148+
if (typeStr.isPresent()) {
149+
String type = typeStr.get();
150+
151+
if ("function_call".equals(type)) {
152+
// Extract function call details
153+
com.openai.core.JsonValue callIdValue = obj.get("call_id");
154+
com.openai.core.JsonValue nameValue = obj.get("name");
155+
com.openai.core.JsonValue argumentsValue = obj.get("arguments");
156+
157+
String callId = null;
158+
String name = null;
159+
String argumentsStr = null;
160+
161+
if (callIdValue != null) {
162+
Optional<String> opt = callIdValue.asString();
163+
if (opt.isPresent()) {
164+
callId = opt.get();
165+
}
166+
}
167+
if (nameValue != null) {
168+
Optional<String> opt = nameValue.asString();
169+
if (opt.isPresent()) {
170+
name = opt.get();
171+
}
172+
}
173+
if (argumentsValue != null) {
174+
Optional<String> opt = argumentsValue.asString();
175+
if (opt.isPresent()) {
176+
argumentsStr = opt.get();
177+
}
178+
}
179+
180+
if (callId != null && name != null && argumentsStr != null) {
181+
Map<String, Object> arguments = parseJsonString(argumentsStr);
182+
LLMObs.ToolCall toolCall =
183+
LLMObs.ToolCall.from(name, "function_call", callId, arguments);
184+
return LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall));
185+
}
186+
} else if ("function_call_output".equals(type)) {
187+
// Extract function call output
188+
com.openai.core.JsonValue callIdValue = obj.get("call_id");
189+
com.openai.core.JsonValue outputValue = obj.get("output");
190+
191+
String callId = null;
192+
String output = null;
193+
194+
if (callIdValue != null) {
195+
Optional<String> opt = callIdValue.asString();
196+
if (opt.isPresent()) {
197+
callId = opt.get();
198+
}
199+
}
200+
if (outputValue != null) {
201+
Optional<String> opt = outputValue.asString();
202+
if (opt.isPresent()) {
203+
output = opt.get();
204+
}
205+
}
206+
207+
if (callId != null && output != null) {
208+
LLMObs.ToolResult toolResult =
209+
LLMObs.ToolResult.from("", "function_call_output", callId, output);
210+
return LLMObs.LLMMessage.fromToolResults("user", Collections.singletonList(toolResult));
211+
}
212+
}
213+
}
214+
}
215+
216+
// Otherwise, it's a regular message with role and content
217+
com.openai.core.JsonValue roleValue = obj.get("role");
218+
com.openai.core.JsonValue contentValue = obj.get("content");
219+
220+
String role = null;
221+
String content = null;
222+
223+
if (roleValue != null) {
224+
Optional<String> opt = roleValue.asString();
225+
if (opt.isPresent()) {
226+
role = opt.get();
227+
}
228+
}
229+
if (contentValue != null) {
230+
Optional<String> opt = contentValue.asString();
231+
if (opt.isPresent()) {
232+
content = opt.get();
233+
}
234+
}
235+
236+
if (role != null) {
237+
return LLMObs.LLMMessage.from(role, content);
238+
}
239+
240+
return null;
241+
}
242+
243+
private Map<String, Object> parseJsonString(String jsonStr) {
244+
if (jsonStr == null || jsonStr.isEmpty()) {
245+
return Collections.emptyMap();
246+
}
247+
try {
248+
jsonStr = jsonStr.trim();
249+
if (!jsonStr.startsWith("{") || !jsonStr.endsWith("}")) {
250+
return Collections.emptyMap();
251+
}
252+
253+
Map<String, Object> result = new HashMap<>();
254+
String content = jsonStr.substring(1, jsonStr.length() - 1).trim();
255+
256+
if (content.isEmpty()) {
257+
return result;
258+
}
259+
260+
// Parse JSON manually, respecting quoted strings
261+
List<String> pairs = splitByCommaRespectingQuotes(content);
262+
263+
for (String pair : pairs) {
264+
int colonIdx = pair.indexOf(':');
265+
if (colonIdx > 0) {
266+
String key = pair.substring(0, colonIdx).trim();
267+
String value = pair.substring(colonIdx + 1).trim();
268+
269+
// Remove quotes from key
270+
key = removeQuotes(key);
271+
// Remove quotes from value
272+
value = removeQuotes(value);
273+
274+
result.put(key, value);
275+
}
276+
}
277+
278+
return result;
279+
} catch (Exception e) {
280+
return Collections.emptyMap();
281+
}
282+
}
283+
284+
private List<String> splitByCommaRespectingQuotes(String str) {
285+
List<String> result = new ArrayList<>();
286+
StringBuilder current = new StringBuilder();
287+
boolean inQuotes = false;
288+
289+
for (int i = 0; i < str.length(); i++) {
290+
char c = str.charAt(i);
291+
292+
if (c == '"') {
293+
inQuotes = !inQuotes;
294+
current.append(c);
295+
} else if (c == ',' && !inQuotes) {
296+
result.add(current.toString());
297+
current = new StringBuilder();
298+
} else {
299+
current.append(c);
300+
}
301+
}
302+
303+
if (current.length() > 0) {
304+
result.add(current.toString());
305+
}
306+
307+
return result;
308+
}
309+
310+
private String removeQuotes(String str) {
311+
str = str.trim();
312+
if (str.startsWith("\"") && str.endsWith("\"") && str.length() >= 2) {
313+
return str.substring(1, str.length() - 1);
314+
}
315+
return str;
316+
}
317+
318+
private String extractInputMessageContent(ResponseInputItem.Message message) {
319+
StringBuilder contentBuilder = new StringBuilder();
320+
for (ResponseInputContent content : message.content()) {
321+
if (content.isInputText()) {
322+
contentBuilder.append(content.asInputText().text());
323+
}
324+
}
325+
String result = contentBuilder.toString();
326+
return result.isEmpty() ? null : result;
327+
}
328+
70329
private Optional<Map<String, String>> extractReasoningFromParams(ResponseCreateParams params) {
71330
com.openai.core.JsonField<Reasoning> reasoningField = params._reasoning();
72331
if (reasoningField.isMissing()) {

‎dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ public ResponseModule() {
1616
public String[] helperClassNames() {
1717
return new String[] {
1818
packageName + ".ResponseDecorator",
19+
packageName + ".FunctionCallOutputExtractor",
1920
packageName + ".OpenAiDecorator",
2021
packageName + ".HttpResponseWrappers",
2122
packageName + ".HttpResponseWrappers$DDHttpResponseFor",

0 commit comments

Comments
 (0)