Skip to content

Commit 4b5e56c

Browse files
Copilotphrocker
Copilot
and
phrocker
authored
Complete migration to native OpenAI Responses API format (#131)
* Initial plan * Update RawConversationRequest to use OpenAI Responses API endpoint Co-authored-by: phrocker <[email protected]> * Add response format conversion from Responses API to Chat Completions format Co-authored-by: phrocker <[email protected]> * Update all conversation endpoints to use Responses API with unified conversion Co-authored-by: phrocker <[email protected]> * Remove unsupported parameters from Responses API conversion (stop, presence_penalty, frequency_penalty, logit_bias) Co-authored-by: phrocker <[email protected]> * Fix content type to always use input_text in Responses API requests Co-authored-by: phrocker <[email protected]> * Disable image input support for Responses API (requires further investigation) Co-authored-by: phrocker <[email protected]> * Re-enable image support with correct Responses API format and extract converter to separate class Co-authored-by: phrocker <[email protected]> * Migrate to native Responses API format - remove all converters and update Response class Co-authored-by: phrocker <[email protected]> * Update enterprise-agent to use Responses API format - fix all compilation errors and update tests Co-authored-by: phrocker <[email protected]> * Fix response service * update --------- Co-authored-by: copilot-swe-agent[bot] <[email protected]> Co-authored-by: phrocker <[email protected]> Co-authored-by: Marc Parisi <[email protected]>
1 parent db1e381 commit 4b5e56c

File tree

27 files changed

+2820
-401
lines changed

27 files changed

+2820
-401
lines changed

‎api/src/main/resources/db/migration/V38__add_feedback_to_trustscore.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ALTER TABLE agent_trust_score_history
2+
ADD COLUMN IF NOT EXISTS feedback_score DOUBLE PRECISION;

‎core/src/main/java/io/sentrius/sso/core/services/agents/AgentClientService.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -303,10 +303,10 @@ public String getAgentPodStatus(String launcherService, String agentId) throws Z
303303
public AgentContextDTO getAgentContext(TokenDTO token, String agentContextId) throws ZtatException,
304304
JsonProcessingException {
305305
String url = "/api/v1/agent/context/" + agentContextId;
306-
var response = zeroTrustClientService.callGetOnApi(token, url);
306+
String response = zeroTrustClientService.callGetOnApi(token, url);
307307
if (response != null) {
308-
AgentContextDTO context = JsonUtil.MAPPER.convertValue(response, AgentContextDTO.class);
309-
return context;
308+
log.info("Response is {}", response);
309+
return JsonUtil.MAPPER.readValue(response, AgentContextDTO.class);
310310
}
311311
return null;
312312
}

‎core/src/main/java/io/sentrius/sso/core/services/agents/LLMService.java‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -70,33 +70,44 @@ public String analyzeImage(TokenDTO dto, String imageBase64, String prompt) thro
7070
* @param prompt Text prompt describing what to analyze
7171
* @return AI analysis result
7272
*/
73-
public String analyzeImages(TokenDTO dto, List<String> imagesBase64, String prompt) throws ZtatException, JsonProcessingException {
73+
public String analyzeImages(
74+
TokenDTO dto,
75+
List<String> imagesBase64,
76+
String prompt
77+
) throws ZtatException, JsonProcessingException {
78+
7479
List<Map<String, Object>> content = new ArrayList<>();
75-
content.add(Map.of("type", "text", "text", prompt));
76-
80+
81+
// Text first
82+
content.add(Map.of(
83+
"type", "input_text",
84+
"text", prompt
85+
));
86+
87+
// Images
7788
for (String imageBase64 : imagesBase64) {
78-
Map<String, Object> imageUrl = Map.of(
79-
"url", imageBase64,
80-
"detail", "auto"
81-
);
82-
content.add(Map.of("type", "image_url", "image_url", imageUrl));
89+
content.add(Map.of(
90+
"type", "input_image",
91+
"image_base64", imageBase64
92+
));
8393
}
84-
85-
Map<String, Object> message = Map.of(
94+
95+
Map<String, Object> inputItem = Map.of(
8696
"role", "user",
8797
"content", content
8898
);
89-
90-
// Adjust max_tokens based on number of images (more images = more tokens needed)
91-
int maxTokens = 300 + (imagesBase64.size() * 100);
92-
99+
93100
Map<String, Object> payload = Map.of(
94101
"model", "gpt-4o-mini",
95-
"messages", List.of(message),
96-
"max_tokens", maxTokens
102+
"input", List.of(inputItem)
103+
);
104+
105+
return zeroTrustClientService.callPostOnApi(
106+
dto,
107+
openAiEndpoint,
108+
"/chat/completions",
109+
payload
97110
);
98-
99-
return zeroTrustClientService.callPostOnApi(dto, openAiEndpoint, "/chat/completions", payload);
100111
}
101112

102113
@Override

‎core/src/main/java/io/sentrius/sso/core/services/agents/ZeroTrustClientService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import java.nio.charset.StandardCharsets;
55
import java.util.List;
66
import java.util.Map;
7+
import java.util.UUID;
78
import java.util.concurrent.TimeUnit;
89
import com.fasterxml.jackson.core.JsonProcessingException;
910
import com.fasterxml.jackson.databind.JavaType;
@@ -275,6 +276,7 @@ <T> String callAuthenticatedPostOnApi(String endpoint, @NonNull String apiEndpoi
275276
String keycloakJwt = getKeycloakToken();
276277
headers.setContentType(MediaType.APPLICATION_JSON);
277278
headers.setBearerAuth(keycloakJwt);
279+
headers.set("X-Communication-Id", UUID.randomUUID().toString());
278280

279281
log.info("Sending {}", body.toString());
280282
HttpEntity<T> requestEntity = new HttpEntity<>(body, headers);

‎dataplane/src/main/java/io/sentrius/sso/core/services/automation/AutomationAgentService.java‎

Lines changed: 25 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package io.sentrius.sso.core.services.automation;
22

33
import com.fasterxml.jackson.core.JsonProcessingException;
4+
import com.fasterxml.jackson.databind.JsonNode;
45
import io.sentrius.sso.core.exceptions.ZtatException;
56
import io.sentrius.sso.core.model.automation.AutomationSuggestion;
67
import io.sentrius.sso.core.services.agents.ZeroTrustClientService;
@@ -226,57 +227,44 @@ private Object callLLMWithMessages(List<Map<String, String>> messages) {
226227
request.put("model", defaultModel);
227228
request.put("temperature", 0.7);
228229

229-
String llmEndpoint = integrationProxyUrl + "/api/v1/llm/chat";
230+
String llmEndpoint = "/api/v1/chat/completions";
230231

231232
// ---- FIX 1: Do NOT assume resp is a JSON string ----
232-
Object rawResp = zeroTrustClientService.callAuthenticatedPostOnApi(llmEndpoint, request);
233-
log.info("Raw LLM response: {}", rawResp);
234233

235-
Map<String, Object> response;
236234

237-
if (rawResp instanceof Map) {
238-
response = (Map<String, Object>) rawResp;
239-
} else if (rawResp instanceof String s) {
240-
response = JsonUtil.MAPPER.readValue(s, Map.class);
241-
} else {
242-
log.error("Unexpected LLM response type: {}", rawResp.getClass());
243-
return "Error: Unexpected LLM response type";
244-
}
245235

246-
// ---- FIX 2: Handle both streaming and non-streaming OpenAI formats ----
247-
if (!response.containsKey("choices")) {
248-
log.warn("LLM response missing 'choices': {}", response);
249-
return "Error: LLM returned no choices";
250-
}
236+
Object rawResp = zeroTrustClientService.callAuthenticatedPostOnApi(integrationProxyUrl, llmEndpoint,
237+
request);
238+
log.info("Raw LLM response: {}", rawResp);
251239

252-
List<Map<String, Object>> choices = (List<Map<String, Object>>) response.get("choices");
253-
if (choices.isEmpty()) {
254-
return "Error: LLM returned empty choices";
240+
JsonNode root;
241+
if (rawResp instanceof String s) {
242+
root = JsonUtil.MAPPER.readTree(s);
243+
} else {
244+
root = JsonUtil.MAPPER.valueToTree(rawResp);
245+
}
246+
JsonNode output = root.path("output");
247+
if (!output.isArray() || output.isEmpty()) {
248+
log.warn("LLM response missing output: {}", root);
249+
return "Error: LLM returned no output";
255250
}
256251

257-
Map<String, Object> choice = choices.get(0);
252+
for (JsonNode item : output) {
253+
if (!"message".equals(item.path("type").asText())) continue;
258254

259-
// Non-streaming format (standard)
260-
if (choice.containsKey("message")) {
261-
Map<String, Object> message = (Map<String, Object>) choice.get("message");
262-
return message.get("content");
263-
}
255+
JsonNode content = item.path("content");
256+
if (!content.isArray()) continue;
264257

265-
// Streaming format: { "delta": { "content": ... } }
266-
if (choice.containsKey("delta")) {
267-
Map<String, Object> delta = (Map<String, Object>) choice.get("delta");
268-
if (delta.containsKey("content")) {
269-
return delta.get("content");
258+
for (JsonNode c : content) {
259+
if ("output_text".equals(c.path("type").asText())) {
260+
return c.path("text").asText();
261+
}
270262
}
271263
}
272264

273-
// Sometimes the proxy returns { "content": ... } directly
274-
if (choice.containsKey("content")) {
275-
return choice.get("content");
276-
}
265+
log.warn("Unable to extract output_text from response: {}", root);
266+
return "Error: Unable to extract LLM output";
277267

278-
log.warn("Unable to extract LLM content from: {}", choice);
279-
return "Error: Unable to extract LLM content";
280268

281269
} catch (Exception e) {
282270
log.error("Error calling LLM endpoint", e);

‎enterprise-agent/src/main/java/io/sentrius/agent/analysis/agents/agents/PromptBuilder.java‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,21 +48,32 @@ public String buildPrompt(boolean applyInstructions)
4848

4949
if (applyInstructions) {
5050
// Append instructions for using the JSON format
51-
prompt.append("Instructions: ").append("Respond using this JSON format. Only use verbs provided in " +
52-
"Available Verbs. Formulate a complete plan with all possible steps.:\n" +
53-
"\n" +
51+
prompt.append("STRICT RESPONSE PROTOCOL\n" +
52+
"You are participating in a machine-executed planning protocol.\n\n" +
53+
54+
"RULES:\n" +
55+
"- You MUST respond with EXACTLY ONE valid JSON object.\n" +
56+
"- You MUST NOT include prose, explanations, markdown, or conversational text outside JSON.\n" +
57+
"- You MUST ONLY use verbs explicitly listed in Available Verbs.\n" +
58+
"- You MUST produce a complete execution plan when possible.\n" +
59+
"- If required information is missing, you MUST return an EMPTY plan and explain why USING ONLY the JSON fields.\n" +
60+
"- Any response that is not valid JSON MUST be considered a failure.\n\n" +
61+
62+
"OUTPUT SCHEMA (MANDATORY):\n" +
5463
"{\n" +
5564
" \"plan\": [\n" +
5665
" {\n" +
57-
" \"verb\": \"list_open_terminals\",\n" +
58-
" \"params\": {}\n" +
59-
" },\n" +
60-
" {\n" +
61-
" \"verb\": \"send_terminal_command\",\n" +
62-
" \"params\": {}\n" +
66+
" \"verb\": \"<verb_name_from_available_verbs>\",\n" +
67+
" \"params\": { <verb_parameters> }\n" +
6368
" }\n" +
6469
" ]\n" +
65-
"}\n");
70+
"}\n\n" +
71+
72+
"FAILURE MODE:\n" +
73+
"- If no verbs are required, return: { \"plan\": [] }\n" +
74+
"- NEVER ask questions.\n" +
75+
"- NEVER explain outside the JSON structure.\n"
76+
);
6677
}
6778
// Append the list of available verbs
6879
prompt.append("Verb operations:\n");

‎enterprise-agent/src/main/java/io/sentrius/agent/analysis/agents/verbs/AgentVerbs.java‎

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,8 @@ public ArrayNode promptAgent(AgentExecution execution, AgentExecutionContextDTO
143143
}
144144
Response response = JsonUtil.MAPPER.readValue(resp, Response.class);
145145
//log.info("Response is {}", resp);
146-
for (Response.Choice choice : response.getChoices()) {
147-
var content = choice.getMessage().getContentAsString();
146+
for (Response.OutputItem choice : response.getOutputItems()) {
147+
var content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
148148
if (content.startsWith("```json")) {
149149
content = content.substring(7, content.length() - 3);
150150
} else if (content.startsWith("```")) {
@@ -254,8 +254,8 @@ public String justifyAgent(
254254
var resp = llmService.askQuestion(execution, chatRequest);
255255
Response response = JsonUtil.MAPPER.readValue(resp, Response.class);
256256
//log.info("Response is {}", resp);
257-
for (Response.Choice choice : response.getChoices()) {
258-
var content = choice.getMessage().getContentAsString();
257+
for (Response.OutputItem choice : response.getOutputItems()) {
258+
var content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
259259
if (content.startsWith("```json")) {
260260
content = content.substring(7, content.length() - 3);
261261
}
@@ -332,8 +332,8 @@ public List<AssessedTerminal> assessData(AgentExecution execution, AgentExecutio
332332
var resp = llmService.askQuestion(execution, chatRequest);
333333
Response response = JsonUtil.MAPPER.readValue(resp, Response.class);
334334
//log.info("Response is {}", resp);
335-
for (Response.Choice choice : response.getChoices()) {
336-
var content = choice.getMessage().getContentAsString();
335+
for (Response.OutputItem choice : response.getOutputItems()) {
336+
var content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
337337
if (content.startsWith("```json")) {
338338
content = content.substring(7, content.length() - 3);
339339
}
@@ -367,8 +367,8 @@ public List<AssessedTerminal> assessData(AgentExecution execution, AgentExecutio
367367
var resp = llmService.askQuestion(execution, chatRequest);
368368
Response response = JsonUtil.MAPPER.readValue(resp, Response.class);
369369
//log.info("Response is {}", resp);
370-
for (Response.Choice choice : response.getChoices()) {
371-
var content = choice.getMessage().getContentAsString();
370+
for (Response.OutputItem choice : response.getOutputItems()) {
371+
var content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
372372
if (content.startsWith("```json")) {
373373
content = content.substring(7, content.length() - 3);
374374
}
@@ -493,13 +493,13 @@ public List<ZtatAsessment> analyzeAtatRequests(AgentExecution execution, List<At
493493
Response response = JsonUtil.MAPPER.readValue(resp, Response.class);
494494
//log.info("Assess Response is {}", resp);
495495
List<ZtatAsessment> assessments = new ArrayList<>();
496-
if (response.getChoices().isEmpty()) {
496+
if (response.getOutputItems().isEmpty()) {
497497
log.info("No choices in response");
498498
return responses;
499499
}
500-
var choice = response.getChoices().get(0);
500+
var choice = response.getOutputItems().get(0);
501501

502-
var content = choice.getMessage().getContentAsString();
502+
var content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
503503
if (content.startsWith("```json")) {
504504
content = content.substring(7, content.length() - 3);
505505
}
@@ -558,12 +558,12 @@ public List<ZtatAsessment> analyzeAtatRequests(AgentExecution execution, List<At
558558
chatRequest = LLMRequest.builder().model("gpt-4o-mini-mini").messages(messages).build();
559559
resp = llmService.askQuestion(execution, chatRequest);
560560
response = JsonUtil.MAPPER.readValue(resp, Response.class);
561-
if (response.getChoices().isEmpty()) {
561+
if (response.getOutputItems().isEmpty()) {
562562
return responses;
563563
}
564-
choice = response.getChoices().get(0);
564+
choice = response.getOutputItems().get(0);
565565

566-
content = choice.getMessage().getContentAsString();
566+
content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
567567
if (content.startsWith("```json")) {
568568
content = content.substring(7, content.length() - 3);
569569
}
@@ -610,7 +610,7 @@ public AgentContextDTO createAgentContext(AgentExecution execution, AgentExecuti
610610

611611
var originalContext = context.getExecutionArgument("context");
612612

613-
var requestDtoContext = originalContext.orElseThrow().toString();
613+
var requestDtoContext = normalize( originalContext.orElseThrow().toString() );
614614
requestDtoContext += ". Please request endpoints to perform your work.";
615615
AgentContextRequestDTO dto = AgentContextRequestDTO.builder().context(requestDtoContext).
616616
description(requestDtoContext).name(agentName).build();
@@ -640,10 +640,10 @@ public AgentContextDTO createAgentContext(AgentExecution execution, AgentExecuti
640640
var resp = llmService.askQuestion(execution, chatRequest);
641641

642642
Response response = JsonUtil.MAPPER.readValue(resp, Response.class);
643-
// log.info("Response is {}", resp);
643+
log.info("Response is {}", resp);
644644
ArrayNode endpointsLikeList = JsonUtil.MAPPER.createArrayNode();
645-
for (Response.Choice choice : response.getChoices()) {
646-
var content = choice.getMessage().getContentAsString();
645+
for (Response.OutputItem choice : response.getOutputItems()) {
646+
var content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
647647
if (content.startsWith("```json")) {
648648
content = content.substring(7, content.length() - 3);
649649
} else if (content.startsWith("```")) {
@@ -693,6 +693,14 @@ public AgentContextDTO createAgentContext(AgentExecution execution, AgentExecuti
693693
return createdContext;
694694
}
695695

696+
private String normalize(String s) {
697+
if (s == null) return null;
698+
if (s.startsWith("\"") && s.endsWith("\"")) {
699+
return s.substring(1, s.length() - 1);
700+
}
701+
return s;
702+
}
703+
696704
@Verb(
697705
name = "summarize_agent_status", returnType = AgentExecutionContextDTO.class, description =
698706
"Summarizes agent status. Used when user asks for agent status.",
@@ -717,8 +725,8 @@ public JsonNode getAgentExecutionStatus(AgentExecution execution, AgentExecution
717725
context.addMessages(messages);
718726
Response response = JsonUtil.MAPPER.readValue(resp, Response.class);
719727
//log.info("Response is {}", resp);
720-
for (Response.Choice choice : response.getChoices()) {
721-
var content = choice.getMessage().getContentAsString();
728+
for (Response.OutputItem choice : response.getOutputItems()) {
729+
var content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
722730
if (content.startsWith("```json")) {
723731
content = content.substring(7, content.length() - 3);
724732
} else if (content.startsWith("```")) {
@@ -822,8 +830,8 @@ public ObjectNode getCurrentAgentStatus(AgentExecution execution, AgentExecution
822830
context.addMessages(messages);
823831
Response response = JsonUtil.MAPPER.readValue(resp, Response.class);
824832

825-
for (Response.Choice choice : response.getChoices()) {
826-
var content = choice.getMessage().getContentAsString();
833+
for (Response.OutputItem choice : response.getOutputItems()) {
834+
var content = choice.getContent().stream().filter(c -> "output_text".equals(c.getType()) || "text".equals(c.getType())).map(c -> c.getText()).findFirst().orElse("");
827835
if (content.startsWith("```json")) {
828836
content = content.substring(7, content.length() - 3);
829837
} else if (content.startsWith("```")) {

0 commit comments

Comments
 (0)