Skip to content

Commit 7633846

Browse files
feat(acp): record response latency and expose agent metadata
- Add elapsed time tracking to _record_usage() via add_response_latency() for both main prompt and fork (ask_agent) paths - Capture agent_version from InitializeResponse.agent_info alongside agent_name - Expose agent_name and agent_version as public properties on ACPAgent so benchmarks can include ACP server identity in eval output Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 3d9803a commit 7633846

File tree

1 file changed

+41
-8
lines changed

1 file changed

+41
-8
lines changed

openhands-sdk/openhands/sdk/agent/acp_agent.py

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -418,11 +418,23 @@ class ACPAgent(AgentBase):
418418
_closed: bool = PrivateAttr(default=False)
419419
_working_dir: str = PrivateAttr(default="")
420420
_agent_name: str = PrivateAttr(default="") # ACP server name from InitializeResponse
421+
_agent_version: str = PrivateAttr(default="") # ACP server version from InitializeResponse
421422

422423
# -- Helpers -----------------------------------------------------------
423424

424-
def _record_usage(self, response: PromptResponse | None, session_id: str) -> None:
425-
"""Record token usage and notify stats callback from a PromptResponse."""
425+
def _record_usage(
426+
self,
427+
response: PromptResponse | None,
428+
session_id: str,
429+
elapsed: float | None = None,
430+
) -> None:
431+
"""Record token usage, latency, and notify stats callback from a PromptResponse.
432+
433+
Args:
434+
response: The ACP PromptResponse (may carry a ``usage`` field).
435+
session_id: Session identifier used as the response_id for metrics.
436+
elapsed: Wall-clock seconds for this prompt round-trip (optional).
437+
"""
426438
if response is not None and response.usage is not None:
427439
usage = response.usage
428440
self.llm.metrics.add_token_usage(
@@ -435,6 +447,9 @@ def _record_usage(self, response: PromptResponse | None, session_id: str) -> Non
435447
response_id=session_id,
436448
)
437449

450+
if elapsed is not None:
451+
self.llm.metrics.add_response_latency(elapsed, session_id)
452+
438453
if self.llm.telemetry._stats_update_callback is not None:
439454
try:
440455
self.llm.telemetry._stats_update_callback()
@@ -447,6 +462,16 @@ def _record_usage(self, response: PromptResponse | None, session_id: str) -> Non
447462
def system_message(self) -> str:
448463
return "ACP-managed agent"
449464

465+
@property
466+
def agent_name(self) -> str:
467+
"""Name of the ACP server (from InitializeResponse.agent_info)."""
468+
return self._agent_name
469+
470+
@property
471+
def agent_version(self) -> str:
472+
"""Version of the ACP server (from InitializeResponse.agent_info)."""
473+
return self._agent_version
474+
450475
def get_all_llms(self) -> Generator[LLM, None, None]:
451476
yield self.llm
452477

@@ -527,7 +552,7 @@ def _start_acp_server(self, state: ConversationState) -> None:
527552

528553
working_dir = str(state.workspace.working_dir)
529554

530-
async def _init() -> tuple[Any, Any, Any, str, str]:
555+
async def _init() -> tuple[Any, Any, Any, str, str, str]:
531556
# Spawn the subprocess directly so we can install a
532557
# filtering reader that skips non-JSON-RPC lines some
533558
# ACP servers (e.g. claude-code-acp v0.1.x) write to
@@ -560,9 +585,15 @@ async def _init() -> tuple[Any, Any, Any, str, str]:
560585
# Initialize the protocol and discover server identity
561586
init_response = await conn.initialize(protocol_version=1)
562587
agent_name = ""
588+
agent_version = ""
563589
if init_response.agent_info is not None:
564590
agent_name = init_response.agent_info.name or ""
565-
logger.info("ACP server initialized: agent_name=%r", agent_name)
591+
agent_version = init_response.agent_info.version or ""
592+
logger.info(
593+
"ACP server initialized: agent_name=%r, agent_version=%r",
594+
agent_name,
595+
agent_version,
596+
)
566597

567598
# Authenticate if the server requires it. Some ACP servers
568599
# (e.g. codex-acp) require an explicit authenticate call
@@ -599,10 +630,10 @@ async def _init() -> tuple[Any, Any, Any, str, str]:
599630
mode_id=mode_id, session_id=session_id
600631
)
601632

602-
return conn, process, filtered_reader, session_id, agent_name
633+
return conn, process, filtered_reader, session_id, agent_name, agent_version
603634

604635
result = self._executor.run_async(_init)
605-
self._conn, self._process, self._filtered_reader, self._session_id, self._agent_name = result
636+
self._conn, self._process, self._filtered_reader, self._session_id, self._agent_name, self._agent_version = result
606637
self._working_dir = working_dir
607638

608639
def step(
@@ -658,7 +689,7 @@ async def _prompt() -> PromptResponse:
658689
elapsed = time.monotonic() - t0
659690
logger.info("ACP prompt returned in %.1fs", elapsed)
660691

661-
self._record_usage(response, self._session_id or "")
692+
self._record_usage(response, self._session_id or "", elapsed=elapsed)
662693

663694
# Emit ACPToolCallEvents for each accumulated tool call
664695
for tc in self._client.accumulated_tool_calls:
@@ -786,14 +817,16 @@ async def _fork_and_prompt() -> str:
786817
client._fork_session_id = fork_session_id
787818
client._fork_accumulated_text.clear()
788819
try:
820+
fork_t0 = time.monotonic()
789821
response = await self._conn.prompt(
790822
[text_block(question)],
791823
fork_session_id,
792824
)
793825
await _drain_notifications()
826+
fork_elapsed = time.monotonic() - fork_t0
794827

795828
result = "".join(client._fork_accumulated_text)
796-
self._record_usage(response, fork_session_id)
829+
self._record_usage(response, fork_session_id, elapsed=fork_elapsed)
797830
return result
798831
finally:
799832
client._fork_session_id = None

0 commit comments

Comments
 (0)