feat(bench): HarnessBench 评测适配层——generic_cli 桥接、harness/bare 双模式与机器审计

.github/workflows/java-regression.yml

@@ -51,7 +51,7 @@ jobs:
           CHANGED_FILES="$(git diff --name-only "$BASE" "$HEAD")"
           printf '%s\n' "$CHANGED_FILES"
 
-          if printf '%s\n' "$CHANGED_FILES" | grep -Eq '^(pom\.xml|ai4j-extension-api/|ai4j-plugin-ask-user/|ai4j/|ai4j-agent/|ai4j-coding/|ai4j-cli/|ai4j-spring-boot-starter/|ai4j-flowgram-spring-boot-starter/|ai4j-flowgram-demo/|ai4j-bom/|\.github/workflows/java-regression\.yml$)'; then
+          if printf '%s\n' "$CHANGED_FILES" | grep -Eq '^(pom\.xml|ai4j-extension-api/|ai4j-plugin-ask-user/|ai4j/|ai4j-agent/|ai4j-harness/|ai4j-coding/|ai4j-cli/|ai4j-spring-boot-starter/|ai4j-flowgram-spring-boot-starter/|ai4j-flowgram-demo/|ai4j-bom/|\.github/workflows/java-regression\.yml$)'; then
             echo "java=true" >> "$GITHUB_OUTPUT"
           else
             echo "java=false" >> "$GITHUB_OUTPUT"
@@ -125,6 +125,7 @@ jobs:
           - ai4j-plugin-ask-user
           - ai4j
           - ai4j-agent
+          - ai4j-harness
           - ai4j-coding
           - ai4j-cli
           - ai4j-spring-boot-starter

.gitignore

@@ -78,3 +78,10 @@ javadoc-ai4j.log
 !AGENTS.md
 .harness/
 harness/
+
+# The ai4j-harness module uses a package named "harness"; keep its source
+# and tests trackable even though the private HA ledger is also named harness.
+!/ai4j-harness/src/main/java/io/github/lnyocly/ai4j/harness/
+!/ai4j-harness/src/main/java/io/github/lnyocly/ai4j/harness/**
+!/ai4j-harness/src/test/java/io/github/lnyocly/ai4j/harness/
+!/ai4j-harness/src/test/java/io/github/lnyocly/ai4j/harness/**

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/Agent.java

@@ -35,6 +35,16 @@ public AgentResult run(AgentRequest request) throws Exception {
         return runtime.run(baseContext, request);
     }
 
+    /**
+     * Returns the immutable-by-convention base context used by this Agent.
+     * Harness integrations use {@link AgentContext#toBuilder()} to create a
+     * per-execution overlay; callers should not mutate the returned context in
+     * place while a run is active.
+     */
+    public AgentContext getContext() {
+        return baseContext;
+    }
+
     public void runStream(AgentRequest request, AgentListener listener) throws Exception {
         runtime.runStream(baseContext, request, listener);
     }
@@ -44,23 +54,58 @@ public AgentResult runStreamResult(AgentRequest request, AgentListener listener)
     }
 
     public AgentSession newSession() {
-        return newSession(AgentSessionMetadata.create(), null);
+        return newSession(baseContext, AgentSessionMetadata.create(), null);
     }
 
-    private AgentSession newSession(AgentSessionMetadata metadata, String runId) {
-        AgentMemory memory = memorySupplier == null ? baseContext.getMemory() : memorySupplier.get();
+    /**
+     * Creates a session from a context overlay while retaining the Agent's
+     * configured memory supplier and runtime. This is intentionally additive:
+     * existing callers continue to use {@link #newSession()}.
+     */
+    public AgentSession newSessionWithContext(AgentContext context) {
+        return newSession(context == null ? baseContext : context, AgentSessionMetadata.create(), null);
+    }
+
+    /**
+     * Creates a fresh session with a host-selected stable identity. This is
+     * useful for business conversations whose id is owned by the application;
+     * it does not bind that identity to a Harness task.
+     */
+    public AgentSession newSessionWithIdentity(String sessionId,
+                                               String runId,
+                                               AgentContext context) {
+        AgentSessionMetadata metadata = new AgentSessionMetadata(sessionId, 0L, 0L, null);
+        return newSession(context == null ? baseContext : context, metadata, runId);
+    }
+
+    private AgentSession newSession(AgentContext context, AgentSessionMetadata metadata, String runId) {
+        AgentContext sourceContext = context == null ? baseContext : context;
+        if (sourceContext == null) {
+            throw new IllegalStateException("agent context is required");
+        }
+        AgentMemory memory = memorySupplier == null ? sourceContext.getMemory() : memorySupplier.get();
         AgentSessionMetadata sessionMetadata = metadata == null ? AgentSessionMetadata.create() : metadata.copy();
         AgentSessionEventLog eventLog = new InMemoryAgentSessionEventLog();
-        AgentContext sessionContext = baseContext.toBuilder()
+        AgentContext sessionContext = sourceContext.toBuilder()
                 .memory(memory)
                 .sessionId(sessionMetadata.getSessionId())
-                .eventPublisher(sessionEventPublisher(eventLog))
+                .eventPublisher(sessionEventPublisher(eventLog, sourceContext))
                 .build();
         return new AgentSession(runtime, sessionContext, sessionMetadata, eventLog, sessionStore, runId);
     }
 
     public AgentSession newSession(AgentSessionSnapshot snapshot) {
+        return newSession(snapshot, baseContext);
+    }
+
+    /**
+     * Restores a session using a per-execution context overlay. The snapshot
+     * remains the source of session state; the overlay only supplies runtime
+     * facilities such as Harness tools, listeners, and bounded options.
+     */
+    public AgentSession newSession(AgentSessionSnapshot snapshot, AgentContext context) {
         AgentSession session = newSession(
+                context == null ? baseContext : context,
                 snapshot == null ? null : snapshot.getMetadata(),
                 snapshot == null ? null : snapshot.getRunId()
         );
@@ -83,8 +128,9 @@ public AgentSessionStore getSessionStore() {
         return sessionStore;
     }
 
-    private AgentEventPublisher sessionEventPublisher(final AgentSessionEventLog eventLog) {
-        AgentEventPublisher basePublisher = baseContext == null ? null : baseContext.getEventPublisher();
+    private AgentEventPublisher sessionEventPublisher(final AgentSessionEventLog eventLog,
+                                                       AgentContext sourceContext) {
+        AgentEventPublisher basePublisher = sourceContext == null ? null : sourceContext.getEventPublisher();
         List<AgentListener> baseListeners = basePublisher == null ? null : basePublisher.getListeners();
         AgentEventPublisher publisher = new AgentEventPublisher(baseListeners);
         publisher.addListener(new AgentListener() {

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/AgentExecutionStatus.java

@@ -0,0 +1,9 @@
+package io.github.lnyocly.ai4j.agent;
+
+/** Structured status for a bounded Agent invocation. */
+public enum AgentExecutionStatus {
+    COMPLETED,
+    WAITING,
+    CONTINUATION_REQUIRED,
+    FAILED
+}

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/AgentRequest.java

@@ -18,6 +18,12 @@ public class AgentRequest {
     public static final String METADATA_KEY_TURN_ID = "turnId";
     public static final String METADATA_KEY_SESSION_ID = "sessionId";
     public static final String METADATA_KEY_EVENT_ID = "eventId";
+    /** Stable Harness scope selected by the host for one execution. */
+    public static final String METADATA_KEY_HARNESS_SCOPE = "harnessScope";
+    /** Durable Harness execution identity injected into an Agent request. */
+    public static final String METADATA_KEY_HARNESS_EXECUTION_ID = "harnessExecutionId";
+    /** Durable Harness Task identity injected when the execution is Task-bound. */
+    public static final String METADATA_KEY_HARNESS_TASK_ID = "harnessTaskId";
 
     private Object input;
 

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/AgentResult.java

@@ -59,6 +59,15 @@ public class AgentResult {
 
     private String currency;
 
+    /** Optional structured status for bounded or asynchronous runs. */
+    private AgentExecutionStatus executionStatus;
+
+    /** Stable operation identity when the run is waiting on asynchronous work. */
+    private String operationId;
+
+    /** Stable wait identity when the host must resume this run later. */
+    private String waitId;
+
     /** Compatibility constructor retained for the pre-cache-accounting result shape. */
     public AgentResult(String runId,
                        String sessionId,

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/AgentSession.java

@@ -114,6 +114,12 @@ public AgentSession putMetadata(String key, Object value) {
         return this;
     }
 
+    /** Replaces an asynchronous tool's pending result in the session memory. */
+    public boolean replaceToolOutput(String callId, String output) {
+        return context != null && context.getMemory() != null
+                && context.getMemory().replaceToolOutput(callId, output);
+    }
+
     public Object getMetadata(String key) {
         return metadata.getAttribute(key);
     }
@@ -264,7 +270,7 @@ private AgentRequest enrichRequest(AgentRequest request, String turnId) {
         if (metadata != null && metadata.getSessionId() != null) {
             metadataMap.put(AgentRequest.METADATA_KEY_SESSION_ID, metadata.getSessionId());
         }
-        metadataMap.put(AgentRequest.METADATA_KEY_RUN_ID, runId);
+        metadataMap.put(AgentRequest.METADATA_KEY_RUN_ID, this.runId);
         if (turnId != null) {
             metadataMap.put(AgentRequest.METADATA_KEY_TURN_ID, turnId);
         }

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/codeact/CodeActPendingToolException.java

@@ -0,0 +1,13 @@
+package io.github.lnyocly.ai4j.agent.codeact;
+
+/**
+ * Internal control-flow signal used to stop a CodeAct program at a pending
+ * asynchronous tool call. The owning CodeExecutor converts it to a
+ * {@link CodeExecutionResult}; it must never escape to an application caller.
+ */
+final class CodeActPendingToolException extends RuntimeException {
+
+    CodeActPendingToolException(String callId) {
+        super("CodeAct tool execution is waiting: " + callId);
+    }
+}

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/codeact/CodeExecutionRequest.java

@@ -21,4 +21,7 @@ public class CodeExecutionRequest {
     private String user;
 
     private Long timeoutMs;
+
+    /** Runtime call id that owns tool calls made from this CodeAct program. */
+    private String parentCallId;
 }

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/codeact/CodeExecutionResult.java

@@ -1,5 +1,6 @@
 package io.github.lnyocly.ai4j.agent.codeact;
 
+import io.github.lnyocly.ai4j.agent.tool.AgentToolExecutionStatus;
 import lombok.Builder;
 import lombok.Data;
 
@@ -13,7 +14,29 @@ public class CodeExecutionResult {
 
     private String error;
 
+    /** Structured lifecycle state for a bounded or asynchronous execution. */
+    private AgentToolExecutionStatus status;
+
+    /** Durable identity of the external operation, when execution is waiting. */
+    private String operationId;
+
+    /** Durable host wait identity, when execution is waiting. */
+    private String waitId;
+
+    /** Inner tool call that caused the CodeAct program to wait. */
+    private String pendingToolCallId;
+
+    /** Outer CodeAct call that owns {@link #pendingToolCallId}. */
+    private String parentCallId;
+
     public boolean isSuccess() {
-        return error == null || error.isEmpty();
+        return !isWaiting()
+                && !AgentToolExecutionStatus.FAILED.equals(status)
+                && !AgentToolExecutionStatus.UNKNOWN.equals(status)
+                && (error == null || error.isEmpty());
+    }
+
+    public boolean isWaiting() {
+        return AgentToolExecutionStatus.WAITING.equals(status);
     }
 }

ai4j-agent/src/main/java/io/github/lnyocly/ai4j/agent/codeact/GraalVmCodeExecutor.java

@@ -2,6 +2,10 @@
 
 import com.alibaba.fastjson2.JSON;
 import io.github.lnyocly.ai4j.agent.tool.AgentToolCall;
+import io.github.lnyocly.ai4j.agent.tool.AgentToolExecution;
+import io.github.lnyocly.ai4j.agent.tool.AgentToolExecutionStatus;
+import io.github.lnyocly.ai4j.agent.tool.AgentToolResult;
+import io.github.lnyocly.ai4j.agent.tool.AsyncToolExecutors;
 import io.github.lnyocly.ai4j.agent.tool.ToolExecutor;
 import org.graalvm.polyglot.Context;
 import org.graalvm.polyglot.HostAccess;
@@ -15,6 +19,7 @@
 import java.io.PrintStream;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
@@ -136,7 +141,7 @@ private CodeExecutionResult executePythonWithGraalPy(CodeExecutionRequest reques
         ByteArrayOutputStream stdoutBytes = new ByteArrayOutputStream();
         ByteArrayOutputStream stderrBytes = new ByteArrayOutputStream();
         ToolExecutor toolExecutor = request.getToolExecutor();
-        ToolBridge toolBridge = new ToolBridge(toolExecutor, request.getUser());
+        ToolBridge toolBridge = new ToolBridge(toolExecutor, request.getUser(), request.getParentCallId());
 
         ProxyExecutable callTool = new ProxyExecutable() {
             @Override
@@ -204,6 +209,11 @@ public Value call() {
             });
 
             Value value = future.get(timeout, TimeUnit.MILLISECONDS);
+            CodeExecutionResult pending = toolBridge.pendingResult(
+                    new String(stdoutBytes.toByteArray(), StandardCharsets.UTF_8));
+            if (pending != null) {
+                return pending;
+            }
             Value fallback = context.getBindings("python").getMember("__codeact_result");
             String resolved = resolveValue(fallback);
             if (resolved == null) {
@@ -220,18 +230,33 @@ public Value call() {
         } catch (IllegalArgumentException e) {
             return null;
         } catch (TimeoutException e) {
+            CodeExecutionResult pending = toolBridge.pendingResult(
+                    new String(stdoutBytes.toByteArray(), StandardCharsets.UTF_8));
+            if (pending != null) {
+                return pending;
+            }
             return CodeExecutionResult.builder()
                     .stdout(new String(stdoutBytes.toByteArray(), StandardCharsets.UTF_8))
                     .error("code execution timeout")
                     .build();
         } catch (ExecutionException e) {
+            CodeExecutionResult pending = toolBridge.pendingResult(
+                    new String(stdoutBytes.toByteArray(), StandardCharsets.UTF_8));
+            if (pending != null) {
+                return pending;
+            }
             Throwable cause = e.getCause() == null ? e : e.getCause();
             log.warn("GraalPy execution failed", cause);
             return CodeExecutionResult.builder()
                     .stdout(new String(stdoutBytes.toByteArray(), StandardCharsets.UTF_8))
                     .error(String.valueOf(cause.getMessage()))
                     .build();
         } catch (Throwable t) {
+            CodeExecutionResult pending = toolBridge.pendingResult(
+                    new String(stdoutBytes.toByteArray(), StandardCharsets.UTF_8));
+            if (pending != null) {
+                return pending;
+            }
             log.warn("GraalPy execution failed", t);
             return CodeExecutionResult.builder()
                     .stdout(new String(stdoutBytes.toByteArray(), StandardCharsets.UTF_8))
@@ -307,10 +332,14 @@ private String filterPolyglotWarnings(String stderr) {
     private static class ToolBridge {
         private final ToolExecutor toolExecutor;
         private final String user;
+        private final String parentCallId;
+        private int invocation;
+        private AgentToolResult pendingResult;
 
-        private ToolBridge(ToolExecutor toolExecutor, String user) {
+        private ToolBridge(ToolExecutor toolExecutor, String user, String parentCallId) {
             this.toolExecutor = toolExecutor;
             this.user = user;
+            this.parentCallId = parentCallId;
         }
 
         @HostAccess.Export
@@ -329,8 +358,64 @@ public String call(String name, Object args) throws Exception {
             AgentToolCall call = AgentToolCall.builder()
                     .name(resolveName(name))
                     .arguments(arguments)
+                    .callId(nextCallId())
+                    .metadata(parentMetadata())
                     .build();
-            return toolExecutor.execute(call);
+            AgentToolExecution execution = AsyncToolExecutors.start(toolExecutor, call);
+            if (execution == null) {
+                return null;
+            }
+            if (execution.isPending()) {
+                pendingResult = normalize(call, execution.getInitialResult());
+                throw new CodeActPendingToolException(call.getCallId());
+            }
+            AgentToolResult result = execution.await();
+            return result == null ? null : result.getOutput();
+        }
+
+        private AgentToolResult normalize(AgentToolCall call, AgentToolResult source) {
+            AgentToolResult result = source == null ? new AgentToolResult() : source;
+            if (result.getName() == null) {
+                result.setName(call.getName());
+            }
+            if (result.getCallId() == null) {
+                result.setCallId(call.getCallId());
+            }
+            if (result.getStatus() == null) {
+                result.setStatus(AgentToolExecutionStatus.WAITING);
+            }
+            return result;
+        }
+
+        private CodeExecutionResult pendingResult(String stdout) {
+            if (pendingResult == null) {
+                return null;
+            }
+            return CodeExecutionResult.builder()
+                    .stdout(stdout)
+                    .result(pendingResult.getOutput())
+                    .error(pendingResult.getError())
+                    .status(AgentToolExecutionStatus.WAITING)
+                    .operationId(pendingResult.getOperationId())
+                    .waitId(pendingResult.getWaitId())
+                    .pendingToolCallId(pendingResult.getCallId())
+                    .parentCallId(parentCallId)
+                    .build();
+        }
+
+        private String nextCallId() {
+            String suffix = String.valueOf(invocation++);
+            return parentCallId == null || parentCallId.trim().isEmpty()
+                    ? "codeact_tool_" + suffix
+                    : parentCallId + ":tool:" + suffix;
+        }
+
+        private Map<String, Object> parentMetadata() {
+            Map<String, Object> metadata = new LinkedHashMap<String, Object>();
+            if (parentCallId != null && !parentCallId.trim().isEmpty()) {
+                metadata.put(AgentToolCall.METADATA_KEY_PARENT_CALL_ID, parentCallId);
+            }
+            return metadata;
         }
 
         private String resolveName(String name) {