Skip to content

refactor(eve): make the tool-loop instrumentation policy-free #2551

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
chadhietala wants to merge 1 commit into from
Draft

refactor(eve): make the tool-loop instrumentation policy-free #2551

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions packages/eve/src/execution/eve-workflow-attributes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,12 @@
*/

import { CHANNEL_CONTEXT_KEY_NAME } from "#context/key-names.js";
import { ChannelRequestIdKey, type SessionTraceSeed } from "#context/keys.js";
import { shouldCaptureInstrumentationContent } from "#harness/instrumentation/content-policy.js";
import { isSampledTrace } from "#tracing/sampled-trace.js";
import { ChannelRequestIdKey } from "#context/keys.js";
import type { EveAttributeValue } from "#runtime/attributes/normalize.js";
import { normalizeChannelAudience } from "#shared/channel-audience.js";
import { isNonEmptyString } from "#shared/guards.js";
import {
legacyTraceContentVisible,
readLegacyTraceId,
readPlanTraceId,
readPlanIsTraceContentVisible,
readPlanChannelKind,
Expand Down Expand Up @@ -115,15 +114,13 @@ export function isWorkflowTraceContentVisible(serializedContext: Record<string,
const channel = serializedContext[CHANNEL_CONTEXT_KEY_NAME] as
| SerializedChannelAdapter
| undefined;
return shouldCaptureInstrumentationContent(normalizeChannelAudience(channel?.audience));
return legacyTraceContentVisible(channel?.audience);
}

export function readSessionTraceId(serializedContext: Record<string, unknown>): string | undefined {
const plan = readPlanFromContext(serializedContext);
if (plan !== undefined) return readPlanTraceId(plan);
const seed = serializedContext["eve.sessionTraceSeed"] as SessionTraceSeed | undefined;
if (seed === undefined || !isSampledTrace(seed)) return undefined;
return isNonEmptyString(seed.traceId) ? seed.traceId : undefined;
return readLegacyTraceId(serializedContext["eve.sessionTraceSeed"]);
}

/**
Expand Down
3 changes: 0 additions & 3 deletions packages/eve/src/execution/settle-cancelled-turn-step.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { callAdapterEventHandler } from "#channel/adapter.js";
import { dispatchStreamEventHooks } from "#context/hook-lifecycle.js";
import { withContextScope } from "#context/run-step.js";
import { deserializeContext, serializeContext } from "#context/serialize.js";
import { ChannelInstrumentationKey } from "#context/keys.js";
import { setChannelContext } from "#execution/channel-context.js";
import {
createDurableSessionState,
Expand Down Expand Up @@ -112,8 +111,6 @@ export async function settleCancelledTurnStep(input: {
};
const emit =
createInstrumentationHandleEvent({
agentName: bundle.turnAgent.id,
channelKind: ctx.get(ChannelInstrumentationKey)?.kind,
handleEvent: baseEmit,
hooks: instrumentation.hooks,
sessionId: session.sessionId,
Expand Down
52 changes: 0 additions & 52 deletions packages/eve/src/harness/instrumentation/content-policy.test.ts
View file
Open in desktop

This file was deleted.

29 changes: 0 additions & 29 deletions packages/eve/src/harness/instrumentation/content-policy.ts
View file
Open in desktop

This file was deleted.

19 changes: 3 additions & 16 deletions packages/eve/src/harness/instrumentation/native-events.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@ describe("createInstrumentationHandleEvent", () => {
},
};
const handleEvent = createInstrumentationHandleEvent({
agentName: "weather",
handleEvent: async (event) => {
order.push(`durable:${event.type}`);
},
Expand Down Expand Up @@ -171,14 +170,8 @@ describe("createInstrumentationHandleEvent", () => {
]);
});

it("carries the dispatch lineage onto every turn a child session starts", async () => {
it("publishes structural turn facts for controls to enrich", async () => {
const events: { readonly type: string }[] = [];
const parentLineage = {
callId: "call-7",
sessionId: "session-1",
subagentName: "researcher",
turnId: "turn-1",
};
const handleEvent = createInstrumentationHandleEvent({
handleEvent: async () => {},
hooks: {
Expand All @@ -187,8 +180,6 @@ describe("createInstrumentationHandleEvent", () => {
events.push(event);
},
},
parentLineage,
rootSessionId: "session-1",
sessionId: "child-1",
})!;

Expand All @@ -198,19 +189,15 @@ describe("createInstrumentationHandleEvent", () => {
expect(events.filter((event) => event.type === "turn.started")).toEqual([
{
idempotencyKey: turnIdempotencyKey("child-1", "child-turn-1"),
parentLineage,
parentTraceContext: undefined,
rootSessionId: "session-1",
rootSessionId: "child-1",
sequence: 0,
sessionId: "child-1",
turnId: "child-turn-1",
type: "turn.started",
},
{
idempotencyKey: turnIdempotencyKey("child-1", "child-turn-2"),
parentLineage,
parentTraceContext: undefined,
rootSessionId: "session-1",
rootSessionId: "child-1",
sequence: 1,
sessionId: "child-1",
turnId: "child-turn-2",
Expand Down
19 changes: 2 additions & 17 deletions packages/eve/src/harness/instrumentation/native-events.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,7 @@ import type {
InstrumentationHooks,
InstrumentationInputRequestedEvent,
InstrumentationInputResolvedEvent,
InstrumentationParentLineage,
InstrumentationPointEvent,
InstrumentationTraceContext,
InstrumentationUsage,
} from "#instrumentation/lifecycle.js";
import {
Expand All @@ -29,18 +27,11 @@ import type { ResolvedInputBatch } from "#harness/input-requests.js";
import { RuntimeActionSettlementTimesKey } from "#harness/runtime-action-settlement-state.js";
import type { HandleEventFn } from "#harness/types.js";
import type { RuntimeActionRequest, RuntimeActionResult } from "#shared/action-types.js";
import type { ChannelAudience } from "#shared/channel-audience.js";

export interface CreateInstrumentationHandleEventInput {
readonly agentName?: string;
readonly channelKind?: string;
readonly channelAudience?: ChannelAudience;
readonly getAttemptScope?: () => InstrumentationAttemptScope | undefined;
readonly handleEvent?: HandleEventFn;
readonly hooks?: InstrumentationHooks;
readonly parentLineage?: InstrumentationParentLineage;
readonly parentTraceContext?: InstrumentationTraceContext;
readonly rootSessionId?: string;
readonly sessionId: string;
readonly turnId?: string;
}
Expand Down Expand Up @@ -293,12 +284,8 @@ function toLifecycleEvent(
switch (event.type) {
case "session.started":
return {
agentName: input.agentName,
channelAudience: input.channelAudience,
channelKind: input.channelKind,
idempotencyKey: sessionIdempotencyKey(input.sessionId),
parentTraceContext: input.parentTraceContext,
rootSessionId: input.rootSessionId ?? input.sessionId,
rootSessionId: input.sessionId,
sessionId: input.sessionId,
type: "session.started",
};
Expand All @@ -321,9 +308,7 @@ function toLifecycleEvent(
case "turn.started":
return {
idempotencyKey: turnIdempotencyKey(input.sessionId, event.data.turnId),
parentLineage: input.parentLineage,
parentTraceContext: input.parentTraceContext,
rootSessionId: input.rootSessionId ?? input.sessionId,
rootSessionId: input.sessionId,
sequence: event.data.sequence,
sessionId: input.sessionId,
turnId: event.data.turnId,
Expand Down
45 changes: 21 additions & 24 deletions packages/eve/src/harness/instrumentation/runtime-context.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ import type { ModelMessage } from "ai";
import { describe, expect, it, vi } from "vitest";

import { ContextContainer, contextStorage } from "#context/container.js";
import { AuthKey, ChannelInstrumentationKey } from "#context/keys.js";
import { AuthKey } from "#context/keys.js";
import type { HarnessEmissionState } from "#harness/emission.js";
import {
buildTelemetryRuntimeContext,
type BuildTelemetryRuntimeContextInput,
} from "#harness/instrumentation/runtime-context.js";
import type { RuntimeContextResolver } from "#tracing/otel-declaration.js";
import type { SessionRuntimeContextResolver } from "#instrumentation/session-plan.js";
import type { HarnessSession } from "#harness/types.js";
import type {
InstrumentationStepStartedEventInput,
Expand Down Expand Up @@ -52,6 +52,8 @@ function build(
return buildTelemetryRuntimeContext({
eveVersion: "0.0.0-test",
authored: { events: {} },
capturesContent: false,
channel: { kind: "unknown", metadata: {} },
emissionState,
environment: "test",
modelInput: { instructions: undefined, messages },
Expand Down Expand Up @@ -161,13 +163,13 @@ describe("buildTelemetryRuntimeContext", () => {

it("reflects the active channel kind and exposes channel metadata to the resolver", () => {
const ctx = new ContextContainer();
ctx.set(ChannelInstrumentationKey, {
kind: "channel:support",
metadata: { triggeringUserId: "U999" },
});

const runtimeContext = contextStorage.run(ctx, () =>
build({
channel: {
kind: "channel:support",
metadata: { triggeringUserId: "U999" },
},
authored: {
events: {
"step.started": (
Expand All @@ -194,7 +196,7 @@ describe("buildTelemetryRuntimeContext", () => {
});
});

it("withholds model content from hosted unknown-audience resolvers", () => {
it("withholds model content when the frozen producer capture is metadata", () => {
let captured: InstrumentationStepStartedEventInput | undefined;

build({
Expand All @@ -211,16 +213,14 @@ describe("buildTelemetryRuntimeContext", () => {
expect(captured?.modelInput).toEqual({ instructions: undefined, messages: [] });
});

it("exposes model content to public-audience resolvers", () => {
it("exposes model content when the frozen producer capture is content", () => {
const ctx = new ContextContainer();
ctx.set(ChannelInstrumentationKey, {
kind: "channel:public",
metadata: { audience: "public" },
});
let captured: InstrumentationStepStartedEventInput | undefined;

contextStorage.run(ctx, () =>
build({
capturesContent: true,
channel: { kind: "channel:public", metadata: { audience: "public" } },
authored: {
events: {
"step.started": (input: InstrumentationStepStartedEventInput) => {
Expand All @@ -239,10 +239,6 @@ describe("buildTelemetryRuntimeContext", () => {
const roles = ["admin"];
const channelMetadata = { nested: { value: "original" }, triggeringUserId: "U999" };
const ctx = new ContextContainer();
ctx.set(ChannelInstrumentationKey, {
kind: "channel:support",
metadata: channelMetadata,
});
ctx.set(AuthKey, {
attributes: { roles },
authenticator: "jwt",
Expand All @@ -253,6 +249,7 @@ describe("buildTelemetryRuntimeContext", () => {
let captured: InstrumentationStepStartedEventInput | undefined;
contextStorage.run(ctx, () =>
build({
channel: { kind: "channel:support", metadata: channelMetadata },
authored: {
events: {
"step.started": (input: InstrumentationStepStartedEventInput) => {
Expand All @@ -276,40 +273,40 @@ describe("buildTelemetryRuntimeContext", () => {

describe("provider runtimeContext resolvers", () => {
it("emits framework keys when a provider resolver returns undefined", () => {
const resolver: RuntimeContextResolver = () => undefined;
const resolver: SessionRuntimeContextResolver = () => undefined;
const runtimeContext = build({ authored: undefined, providerResolvers: [resolver] });

expect(runtimeContext).toEqual(FRAMEWORK_KEYS);
});

it("merges a single provider resolver beneath framework keys", () => {
const resolver: RuntimeContextResolver = () => ({ team: "platform" });
const resolver: SessionRuntimeContextResolver = () => ({ team: "platform" });
const runtimeContext = build({ authored: undefined, providerResolvers: [resolver] });

expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, team: "platform" });
});

it("merges multiple provider resolvers, later ones overriding earlier", () => {
const first: RuntimeContextResolver = () => ({ env: "prod", team: "a" });
const second: RuntimeContextResolver = () => ({ team: "b" });
const first: SessionRuntimeContextResolver = () => ({ env: "prod", team: "a" });
const second: SessionRuntimeContextResolver = () => ({ team: "b" });
const runtimeContext = build({ authored: undefined, providerResolvers: [first, second] });

expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, env: "prod", team: "b" });
});

it("drops reserved eve.* keys from provider resolver results", () => {
const resolver: RuntimeContextResolver = () =>
const resolver: SessionRuntimeContextResolver = () =>
({ "eve.session.id": "override", team: "platform" }) as never;
const runtimeContext = build({ authored: undefined, providerResolvers: [resolver] });

expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, team: "platform" });
});

it("continues when a provider resolver throws", () => {
const failing: RuntimeContextResolver = () => {
const failing: SessionRuntimeContextResolver = () => {
throw new Error("boom");
};
const healthy: RuntimeContextResolver = () => ({ team: "platform" });
const healthy: SessionRuntimeContextResolver = () => ({ team: "platform" });
const runtimeContext = build({ authored: undefined, providerResolvers: [failing, healthy] });

expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, team: "platform" });
Expand All @@ -321,7 +318,7 @@ describe("buildTelemetryRuntimeContext", () => {
});

it("merges provider resolver results alongside the legacy step.started hook", () => {
const resolver: RuntimeContextResolver = () => ({ source: "provider" });
const resolver: SessionRuntimeContextResolver = () => ({ source: "provider" });
const runtimeContext = build({
authored: { events: { "step.started": () => ({ runtimeContext: { source: "legacy" } }) } },
providerResolvers: [resolver],
Expand Down
Loading
Loading