Skip to content

Commit 2d327a5

Browse files
committed
feat(chat): highlight suspicious failure phrases in Activity thinking
- Add parseThinkingSegments() to detect auth/error phrases in agent thinking - Render thinking text with suspicious segments highlighted (amber mark + tooltip) - Apply in Activity block (past reasoning) and Response block (streaming) - Unit tests for patterns and sidebar highlight; bump version to 1.6.3
1 parent 75d2972 commit 2d327a5

File tree

5 files changed

+179
-3
lines changed

5 files changed

+179
-3
lines changed

apps/chat/src/app/agent-thinking-sidebar.spec.tsx

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,25 @@ describe('AgentThinkingSidebar', () => {
120120
expect(screen.getByText('npm install')).toBeTruthy();
121121
});
122122

123+
it('highlights suspicious failure phrases in reasoning activity block', () => {
124+
const storyItems = [
125+
{ id: '1', type: 'stream_start', message: 'Started', timestamp: new Date().toISOString() },
126+
{
127+
id: '2',
128+
type: 'reasoning_start',
129+
message: '',
130+
timestamp: new Date().toISOString(),
131+
details: 'But authentication fails. Trying fallback.',
132+
},
133+
];
134+
render(
135+
<AgentThinkingSidebar isCollapsed={false} onToggle={vi.fn()} storyItems={storyItems} />
136+
);
137+
const mark = document.querySelector('mark[title="Possible failure — check token or access"]');
138+
expect(mark).toBeTruthy();
139+
expect(mark?.textContent).toContain('authentication fails');
140+
});
141+
123142
it('strips leading "Ran " from tool_call when only message is set', () => {
124143
const storyItems = [
125144
{

apps/chat/src/app/agent-thinking-sidebar.tsx

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ import {
1919
toTimestampMs,
2020
type StoryEntry,
2121
} from './agent-thinking-utils';
22+
import {
23+
parseThinkingSegments,
24+
SUSPICIOUS_TOOLTIP,
25+
} from './thinking-failure-patterns';
2226
import {
2327
ACTIVITY_BLOCK_BASE,
2428
ACTIVITY_BLOCK_VARIANTS,
@@ -152,6 +156,37 @@ const BRAIN_COMPLETE_TO_IDLE_MS = 7_000;
152156

153157
const SINGLE_ROW_TYPES = new Set(['stream_start', 'step', 'tool_call', 'file_created']);
154158

159+
const SUSPICIOUS_SEGMENT_CLASS =
160+
'bg-amber-500/25 text-amber-200 border-b border-amber-500/50 rounded-sm px-0.5';
161+
162+
const ThinkingTextWithHighlights = memo(function ThinkingTextWithHighlights({
163+
text,
164+
className,
165+
}: {
166+
text: string;
167+
className?: string;
168+
}) {
169+
const segments = useMemo(() => parseThinkingSegments(text), [text]);
170+
if (segments.length === 0) return null;
171+
return (
172+
<span className={className}>
173+
{segments.map((seg, i) =>
174+
seg.suspicious ? (
175+
<mark
176+
key={i}
177+
className={SUSPICIOUS_SEGMENT_CLASS}
178+
title={SUSPICIOUS_TOOLTIP}
179+
>
180+
{seg.text}
181+
</mark>
182+
) : (
183+
<span key={i}>{seg.text}</span>
184+
)
185+
)}
186+
</span>
187+
);
188+
});
189+
155190
const ActivityBlock = memo(function ActivityBlock({
156191
entry,
157192
isStreaming,
@@ -230,7 +265,9 @@ const ActivityBlock = memo(function ActivityBlock({
230265
</div>
231266
{isThinkingBlock ? (
232267
<div className="mt-0.5 rounded-md bg-background/40 px-2 py-1.5 max-h-32 overflow-y-auto">
233-
<p className={`text-[11px] ${ACTIVITY_MONO}`}>{entry.details}</p>
268+
<p className={`text-[11px] ${ACTIVITY_MONO}`}>
269+
<ThinkingTextWithHighlights text={entry.details ?? ''} />
270+
</p>
234271
</div>
235272
) : (
236273
<div className="mt-0.5">
@@ -840,7 +877,9 @@ export function AgentThinkingSidebar({
840877
Response
841878
</p>
842879
<div className={`${ACTIVITY_MONO} flex-1 min-h-0 overflow-y-auto`}>
843-
{displayThinkingText || (isStreaming ? '…' : '')}
880+
<ThinkingTextWithHighlights
881+
text={displayThinkingText || (isStreaming ? '…' : '')}
882+
/>
844883
<span
845884
ref={thinkingScrollRef}
846885
className="inline-block min-h-0"
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { parseThinkingSegments, SUSPICIOUS_TOOLTIP } from './thinking-failure-patterns';
2+
3+
describe('parseThinkingSegments', () => {
4+
it('returns single non-suspicious segment when no patterns match', () => {
5+
const out = parseThinkingSegments('User wants to get Github username.');
6+
expect(out).toEqual([{ text: 'User wants to get Github username.', suspicious: false }]);
7+
});
8+
9+
it('marks "but ... fails" as suspicious', () => {
10+
const out = parseThinkingSegments(
11+
'User wants to get GIthub username... But authentication fails… Let me try.'
12+
);
13+
const suspicious = out.filter((s) => s.suspicious);
14+
expect(suspicious.length).toBeGreaterThan(0);
15+
expect(suspicious.some((s) => s.text.toLowerCase().includes('authentication fails'))).toBe(true);
16+
});
17+
18+
it('marks "error" as suspicious', () => {
19+
const out = parseThinkingSegments('Something went wrong. Error connecting.');
20+
expect(out.some((s) => s.suspicious && s.text.toLowerCase().includes('error'))).toBe(true);
21+
});
22+
23+
it('marks 401 and 403 as suspicious', () => {
24+
const out = parseThinkingSegments('Got 401 then 403.');
25+
expect(out.filter((s) => s.suspicious).map((s) => s.text)).toContain('401');
26+
expect(out.filter((s) => s.suspicious).map((s) => s.text)).toContain('403');
27+
});
28+
29+
it('marks permission denied and access denied as suspicious', () => {
30+
const out = parseThinkingSegments('Permission denied. Access denied.');
31+
expect(out.some((s) => s.suspicious && s.text.includes('Permission denied'))).toBe(true);
32+
expect(out.some((s) => s.suspicious && s.text.includes('Access denied'))).toBe(true);
33+
});
34+
35+
it('returns non-suspicious leading and trailing segments around one match', () => {
36+
const out = parseThinkingSegments('Before. authentication fails After.');
37+
expect(out[0]).toEqual({ text: 'Before. ', suspicious: false });
38+
expect(out[1]).toEqual({ text: 'authentication fails', suspicious: true });
39+
expect(out[2]).toEqual({ text: ' After.', suspicious: false });
40+
});
41+
42+
it('returns empty array for empty or whitespace input', () => {
43+
expect(parseThinkingSegments('')).toEqual([]);
44+
expect(parseThinkingSegments(' ')).toEqual([]);
45+
});
46+
47+
it('returns empty array for null or undefined input', () => {
48+
expect(parseThinkingSegments(null as unknown as string)).toEqual([]);
49+
expect(parseThinkingSegments(undefined as unknown as string)).toEqual([]);
50+
});
51+
});
52+
53+
describe('SUSPICIOUS_TOOLTIP', () => {
54+
it('is a non-empty string', () => {
55+
expect(typeof SUSPICIOUS_TOOLTIP).toBe('string');
56+
expect(SUSPICIOUS_TOOLTIP.length).toBeGreaterThan(0);
57+
});
58+
});
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/**
2+
* Detects phrases in agent thinking that suggest failure (e.g. auth, errors).
3+
* Used to highlight suspicious segments in the Activity tab so the user can act (e.g. check token).
4+
*/
5+
6+
export type ThinkingSegment = { text: string; suspicious: boolean };
7+
8+
const SUSPICIOUS_PATTERNS: RegExp[] = [
9+
/\b(but\s+[\w\s]+?\s+fails?)\b/gi,
10+
/\b(authentication\s+fails?|auth\s+fails?)\b/gi,
11+
/\b(failed|fails?)\b/gi,
12+
/\b(error|errors?)\b/gi,
13+
/\b(couldn't|could not|can't|cannot)\s+(\w[\w\s]*?)(?=[.!]|$)/gi,
14+
/\b(unable to\s+\w[\w\s]*?)(?=[.!]|$)/gi,
15+
/\b(permission denied|access denied|not authenticated)\b/gi,
16+
/\b(invalid token|token expired|token invalid)\b/gi,
17+
/\b(401|403)\b/g,
18+
];
19+
20+
function collectRanges(text: string): { start: number; end: number }[] {
21+
const ranges: { start: number; end: number }[] = [];
22+
for (const re of SUSPICIOUS_PATTERNS) {
23+
const copy = new RegExp(re.source, re.flags);
24+
let m: RegExpExecArray | null;
25+
while ((m = copy.exec(text)) !== null) {
26+
const start = m.index;
27+
const end = start + m[0].length;
28+
if (!ranges.some((r) => start < r.end && end > r.start)) ranges.push({ start, end });
29+
}
30+
}
31+
ranges.sort((a, b) => a.start - b.start);
32+
const merged: { start: number; end: number }[] = [];
33+
for (const r of ranges) {
34+
const last = merged[merged.length - 1];
35+
if (last && r.start <= last.end) last.end = Math.max(last.end, r.end);
36+
else merged.push({ start: r.start, end: r.end });
37+
}
38+
return merged;
39+
}
40+
41+
export function parseThinkingSegments(text: string): ThinkingSegment[] {
42+
if (typeof text !== 'string') return [];
43+
const trimmed = text.trim();
44+
if (!trimmed) return [];
45+
46+
const ranges = collectRanges(trimmed);
47+
if (ranges.length === 0) return [{ text: trimmed, suspicious: false }];
48+
49+
const segments: ThinkingSegment[] = [];
50+
let pos = 0;
51+
for (const { start, end } of ranges) {
52+
if (start > pos) segments.push({ text: trimmed.slice(pos, start), suspicious: false });
53+
segments.push({ text: trimmed.slice(start, end), suspicious: true });
54+
pos = end;
55+
}
56+
if (pos < trimmed.length) segments.push({ text: trimmed.slice(pos), suspicious: false });
57+
return segments;
58+
}
59+
60+
export const SUSPICIOUS_TOOLTIP = 'Possible failure — check token or access';

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "phoenix-chat",
3-
"version": "1.6.2",
3+
"version": "1.6.3",
44
"license": "MIT",
55
"packageManager": "bun@1.3.10",
66
"scripts": {

0 commit comments

Comments
 (0)