Skip to content

Commit f06b15c

Browse files
yujongleecursoragentdevin-ai-integration[bot]
authored
feat: enrich chat search results with full session context (#3959)
* removed outdated related session field in template * chat refactors * small dedupe * feat: enrich chat search results with full session context Add load_session_content command to fs-sync plugin to read session data (meta, memo, transcript, notes) from disk. Hydrate search results with resolved participants, speaker-labeled transcripts, and rendered template output so the chat model receives rich context for referenced sessions. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: address review comment and update snapshot test Co-Authored-By: yujonglee <yujonglee.dev@gmail.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 9b2e6d6 commit f06b15c

File tree

21 files changed

+688
-45
lines changed

21 files changed

+688
-45
lines changed

‎apps/desktop/src/chat/context-item.ts‎

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,15 +60,16 @@ function parseSearchSessionsOutput(output: unknown): ContextEntity[] {
6060
return [];
6161
}
6262

63+
const parsedSessionContext = parseSessionContext(item.sessionContext);
6364
const title = typeof item.title === "string" ? item.title : null;
64-
const content = typeof item.content === "string" ? item.content : null;
65+
const content = typeof item.excerpt === "string" ? item.excerpt : null;
6566

6667
return [
6768
{
6869
kind: "session",
6970
key: `session:search:${item.id}`,
7071
source: "tool",
71-
sessionContext: {
72+
sessionContext: parsedSessionContext ?? {
7273
title,
7374
date: null,
7475
rawContent: content,
@@ -83,6 +84,76 @@ function parseSearchSessionsOutput(output: unknown): ContextEntity[] {
8384
});
8485
}
8586

87+
function parseSessionContext(value: unknown): SessionContext | null {
88+
if (!isRecord(value)) {
89+
return null;
90+
}
91+
92+
const title = typeof value.title === "string" ? value.title : null;
93+
const date = typeof value.date === "string" ? value.date : null;
94+
const rawContent =
95+
typeof value.rawContent === "string" ? value.rawContent : null;
96+
const enhancedContent =
97+
typeof value.enhancedContent === "string" ? value.enhancedContent : null;
98+
99+
const participants = Array.isArray(value.participants)
100+
? value.participants.flatMap((participant) => {
101+
if (!isRecord(participant) || typeof participant.name !== "string") {
102+
return [];
103+
}
104+
return [
105+
{
106+
name: participant.name,
107+
jobTitle:
108+
typeof participant.jobTitle === "string"
109+
? participant.jobTitle
110+
: null,
111+
},
112+
];
113+
})
114+
: [];
115+
116+
const event =
117+
isRecord(value.event) && typeof value.event.name === "string"
118+
? { name: value.event.name }
119+
: null;
120+
121+
const transcript = isRecord(value.transcript)
122+
? {
123+
segments: Array.isArray(value.transcript.segments)
124+
? value.transcript.segments.flatMap((segment) => {
125+
if (
126+
!isRecord(segment) ||
127+
typeof segment.speaker !== "string" ||
128+
typeof segment.text !== "string"
129+
) {
130+
return [];
131+
}
132+
return [{ speaker: segment.speaker, text: segment.text }];
133+
})
134+
: [],
135+
startedAt:
136+
typeof value.transcript.startedAt === "number"
137+
? value.transcript.startedAt
138+
: null,
139+
endedAt:
140+
typeof value.transcript.endedAt === "number"
141+
? value.transcript.endedAt
142+
: null,
143+
}
144+
: null;
145+
146+
return {
147+
title,
148+
date,
149+
rawContent,
150+
enhancedContent,
151+
transcript,
152+
participants,
153+
event,
154+
};
155+
}
156+
86157
const toolEntityExtractors: Record<
87158
string,
88159
(output: unknown) => ContextEntity[]

‎apps/desktop/src/chat/session-context-hydrator.ts‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
import { commands as fsSyncCommands } from "@hypr/plugin-fs-sync";
2+
import type { SessionContentData } from "@hypr/plugin-fs-sync";
3+
import type { SessionContext, Transcript } from "@hypr/plugin-template";
4+
import type { SpeakerHintStorage } from "@hypr/store";
5+
import { isValidTiptapContent, json2md } from "@hypr/tiptap/shared";
6+
7+
import type * as main from "../store/tinybase/store/main";
8+
import { buildSegments, SegmentKey, type WordLike } from "../utils/segment";
9+
import {
10+
defaultRenderLabelContext,
11+
SpeakerLabelManager,
12+
} from "../utils/segment/shared";
13+
import { convertStorageHintsToRuntime } from "../utils/speaker-hints";
14+
15+
function toMarkdownFromTiptap(value: unknown): string | null {
16+
if (!value || typeof value !== "object") {
17+
return null;
18+
}
19+
if (!isValidTiptapContent(value)) {
20+
return null;
21+
}
22+
23+
const md = json2md(value);
24+
const trimmed = md.trim();
25+
return trimmed ? trimmed : null;
26+
}
27+
28+
function extractEventName(event: unknown): string | null {
29+
if (!event || typeof event !== "object") {
30+
return null;
31+
}
32+
33+
const record = event as Record<string, unknown>;
34+
if (typeof record.name === "string" && record.name) {
35+
return record.name;
36+
}
37+
if (typeof record.title === "string" && record.title) {
38+
return record.title;
39+
}
40+
41+
return null;
42+
}
43+
44+
function buildTranscript(
45+
transcriptData: SessionContentData["transcript"],
46+
store: ReturnType<typeof main.UI.useStore>,
47+
): Transcript | null {
48+
if (!transcriptData || transcriptData.transcripts.length === 0) {
49+
return null;
50+
}
51+
52+
const indexedWords = transcriptData.transcripts
53+
.flatMap((transcript) =>
54+
transcript.words.map((word) => ({
55+
id: word.id ?? null,
56+
text: word.text,
57+
start_ms: word.startMs,
58+
end_ms: word.endMs,
59+
channel: word.channel as WordLike["channel"],
60+
})),
61+
)
62+
.sort((a, b) => a.start_ms - b.start_ms);
63+
64+
const words: WordLike[] = indexedWords.map((word) => ({
65+
text: word.text,
66+
start_ms: word.start_ms,
67+
end_ms: word.end_ms,
68+
channel: word.channel,
69+
}));
70+
71+
if (words.length === 0) {
72+
return null;
73+
}
74+
75+
const wordIdToIndex = new Map<string, number>();
76+
indexedWords.forEach((word, index) => {
77+
if (typeof word.id === "string" && word.id) {
78+
wordIdToIndex.set(word.id, index);
79+
}
80+
});
81+
82+
const storageHints: SpeakerHintStorage[] = transcriptData.transcripts.flatMap(
83+
(transcript) =>
84+
transcript.speakerHints.flatMap((hint) => {
85+
const start = wordIdToIndex.get(hint.startWordId);
86+
const end = wordIdToIndex.get(hint.endWordId);
87+
if (typeof start !== "number" || typeof end !== "number") {
88+
return [];
89+
}
90+
91+
const from = Math.min(start, end);
92+
const to = Math.max(start, end);
93+
94+
const speakerId = hint.speakerId;
95+
const speakerIndex =
96+
typeof speakerId === "string"
97+
? Number.parseInt(speakerId.replace(/[^\d-]/g, ""), 10)
98+
: Number.NaN;
99+
100+
const isHumanAssignment =
101+
!!store &&
102+
typeof speakerId === "string" &&
103+
Boolean(store.getRow("humans", speakerId));
104+
105+
const type = isHumanAssignment
106+
? "user_speaker_assignment"
107+
: "provider_speaker_index";
108+
const value = JSON.stringify(
109+
isHumanAssignment
110+
? { human_id: speakerId }
111+
: {
112+
speaker_index: Number.isFinite(speakerIndex) ? speakerIndex : 0,
113+
},
114+
);
115+
116+
return indexedWords.slice(from, to + 1).flatMap((word) => {
117+
if (typeof word.id !== "string" || !word.id) {
118+
return [];
119+
}
120+
return [
121+
{
122+
word_id: word.id,
123+
type,
124+
value,
125+
},
126+
];
127+
});
128+
}),
129+
);
130+
131+
const runtimeHints = convertStorageHintsToRuntime(
132+
storageHints,
133+
wordIdToIndex,
134+
);
135+
136+
const segments = buildSegments(words, [], runtimeHints);
137+
const ctx = store ? defaultRenderLabelContext(store) : undefined;
138+
const manager = SpeakerLabelManager.fromSegments(segments, ctx);
139+
140+
const startedAtCandidates = transcriptData.transcripts
141+
.map((t) => t.startedAt)
142+
.filter((v): v is number => typeof v === "number");
143+
const endedAtCandidates = transcriptData.transcripts
144+
.map((t) => t.endedAt)
145+
.filter((v): v is number => typeof v === "number");
146+
147+
return {
148+
segments: segments.map((segment) => ({
149+
speaker: SegmentKey.renderLabel(segment.key, ctx, manager),
150+
text: segment.words.map((word) => word.text).join(" "),
151+
})),
152+
startedAt:
153+
startedAtCandidates.length > 0 ? Math.min(...startedAtCandidates) : null,
154+
endedAt:
155+
endedAtCandidates.length > 0 ? Math.max(...endedAtCandidates) : null,
156+
};
157+
}
158+
159+
export async function hydrateSessionContextFromFs(
160+
store: ReturnType<typeof main.UI.useStore>,
161+
sessionId: string,
162+
): Promise<SessionContext | null> {
163+
const result = await fsSyncCommands.loadSessionContent(sessionId);
164+
if (result.status === "error") {
165+
return null;
166+
}
167+
168+
const payload = result.data;
169+
const participants =
170+
payload.meta?.participants
171+
?.map((participant) => {
172+
const row = store?.getRow("humans", participant.humanId);
173+
if (!row || typeof row.name !== "string" || !row.name) {
174+
return null;
175+
}
176+
177+
return {
178+
name: row.name,
179+
jobTitle:
180+
typeof row.job_title === "string" && row.job_title
181+
? row.job_title
182+
: null,
183+
};
184+
})
185+
.filter(
186+
(
187+
participant,
188+
): participant is { name: string; jobTitle: string | null } =>
189+
Boolean(participant),
190+
) ?? [];
191+
192+
const enhancedContent = payload.notes
193+
.slice()
194+
.sort((a, b) => (a.position ?? 0) - (b.position ?? 0))
195+
.map((note) => toMarkdownFromTiptap(note.tiptapJson))
196+
.filter((note): note is string => Boolean(note))
197+
.join("\n\n---\n\n");
198+
199+
const transcript = buildTranscript(payload.transcript, store);
200+
const eventName = extractEventName(payload.meta?.event);
201+
202+
return {
203+
title: payload.meta?.title ?? null,
204+
date: payload.meta?.createdAt ?? null,
205+
rawContent: toMarkdownFromTiptap(payload.rawMemoTiptapJson),
206+
enhancedContent: enhancedContent || null,
207+
transcript,
208+
participants,
209+
event: eventName ? { name: eventName } : null,
210+
};
211+
}

‎apps/desktop/src/chat/tools.ts‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import { tool } from "ai";
22
import { z } from "zod";
33

4+
import type { SessionContext } from "@hypr/plugin-template";
5+
import { commands as templateCommands } from "@hypr/plugin-template";
6+
47
import { searchFiltersSchema } from "../contexts/search/engine/types";
58
import type { SearchFilters, SearchHit } from "../contexts/search/engine/types";
69
import type { SupportMcpTools } from "./support-mcp-tools";
@@ -10,6 +13,7 @@ export interface ToolDependencies {
1013
query: string,
1114
filters?: SearchFilters | null,
1215
) => Promise<SearchHit[]>;
16+
resolveSessionContext: (sessionId: string) => Promise<SessionContext | null>;
1317
}
1418

1519
const buildSearchSessionsTool = (deps: ToolDependencies) =>
@@ -27,15 +31,31 @@ const buildSearchSessionsTool = (deps: ToolDependencies) =>
2731
execute: async (params: { query: string; filters?: SearchFilters }) => {
2832
const hits = await deps.search(params.query, params.filters || null);
2933

30-
const results = hits.slice(0, 5).map((hit) => ({
31-
id: hit.document.id,
32-
title: hit.document.title,
33-
content: hit.document.content.slice(0, 500),
34-
score: hit.score,
35-
created_at: hit.document.created_at,
34+
const results = await Promise.all(
35+
hits.slice(0, 5).map(async (hit) => ({
36+
id: hit.document.id,
37+
title: hit.document.title,
38+
excerpt: hit.document.content.slice(0, 180),
39+
score: hit.score,
40+
created_at: hit.document.created_at,
41+
sessionContext: await deps.resolveSessionContext(hit.document.id),
42+
})),
43+
);
44+
const templateResults = results.map((result) => ({
45+
...result,
46+
createdAt: result.created_at,
3647
}));
3748

38-
return { results };
49+
const rendered = await templateCommands.render({
50+
toolSearchSessions: {
51+
query: params.query,
52+
results: templateResults,
53+
},
54+
});
55+
56+
const contextText = rendered.status === "ok" ? rendered.data : null;
57+
58+
return { results, contextText };
3959
},
4060
});
4161

@@ -50,10 +70,12 @@ type LocalTools = {
5070
results: Array<{
5171
id: string;
5272
title: string;
53-
content: string;
73+
excerpt: string;
5474
score: number;
5575
created_at: number;
76+
sessionContext: SessionContext | null;
5677
}>;
78+
contextText: string | null;
5779
};
5880
};
5981
};

0 commit comments

Comments
 (0)