Skip to content

Commit 964b4da

Browse files
committed
refactor: replace verbose flag with debug option
- Simplify CLI options by consolidating quiet/verbose into single debug flag - Remove quiet mode functionality while preserving debug output capabilities - Add conversation formatting for debug mode with message display - Improve tool call validation error messages with context
1 parent ea10652 commit 964b4da

File tree

12 files changed

+169
-54
lines changed

12 files changed

+169
-54
lines changed

src/cli.ts

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ interface CliOptions {
1919
serverConfig: string;
2020
serverName?: string;
2121
timeout?: number;
22-
quiet?: boolean;
23-
verbose?: boolean;
22+
debug?: boolean;
2423
junitXml?: string;
2524
}
2625

@@ -113,8 +112,7 @@ Examples:
113112
'Specific server name to use from config (if multiple servers defined)'
114113
)
115114
.option('--timeout <ms>', 'Test timeout in milliseconds', '10000')
116-
.option('--quiet', 'Suppress non-essential output')
117-
.option('--verbose', 'Enable verbose output with additional details')
115+
.option('--debug', 'Enable debug output with additional details')
118116
.option('--junit-xml [filename]', 'Generate JUnit XML output (default: junit.xml)')
119117
.action(async (testFile: string, options: CliOptions) => {
120118
await runTests(testFile, options);

src/testing/capabilities/runner.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ import type { DisplayManager } from '../display/DisplayManager.js';
1919
interface ServerOptions {
2020
serverConfig: ServerConfig;
2121
timeout?: number;
22-
quiet?: boolean;
23-
verbose?: boolean;
22+
debug?: boolean;
2423
}
2524

2625
export class CapabilitiesTestRunner {

src/testing/display/DisplayManager.ts

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,14 @@ export class DisplayManager {
1212
constructor(options: DisplayOptions = {}) {
1313
this.formatters = [];
1414

15-
// Always include console formatter unless quiet mode
16-
if (!options.quiet) {
17-
this.formatters.push(new ConsoleFormatter(options));
18-
}
15+
// Always include console formatter
16+
this.formatters.push(new ConsoleFormatter(options));
1917

2018
// Add JUnit XML formatter if requested
2119
if (options.junitXml !== undefined) {
2220
const filename = options.junitXml || 'junit.xml';
2321
this.formatters.push(new JunitXmlFormatter(options, filename));
2422
}
25-
26-
// Ensure we have at least one formatter
27-
if (this.formatters.length === 0) {
28-
this.formatters.push(new ConsoleFormatter(options));
29-
}
3023
}
3124

3225
/**
@@ -69,11 +62,12 @@ export class DisplayManager {
6962
passed: boolean,
7063
errors: string[],
7164
model?: string,
72-
prompt?: string
65+
prompt?: string,
66+
messages?: any[]
7367
): void {
7468
this.emit({
7569
type: 'test_complete',
76-
data: { name, model, passed, errors, prompt },
70+
data: { name, model, passed, errors, prompt, messages },
7771
});
7872
}
7973

src/testing/display/formatters/ConsoleFormatter.ts

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import type {
1515
TestCompleteEvent,
1616
SuiteCompleteEvent,
1717
} from '../types.js';
18+
import { formatConversation } from '../utils/conversationFormatter.js';
1819

1920
export class ConsoleFormatter implements TestFormatter {
2021
private options: DisplayOptions;
@@ -26,10 +27,6 @@ export class ConsoleFormatter implements TestFormatter {
2627
}
2728

2829
onEvent(event: TestEvent): void {
29-
if (this.options.quiet) {
30-
return;
31-
}
32-
3330
switch (event.type) {
3431
case 'suite_start':
3532
this.handleSuiteStart(event.data as SuiteStartEvent['data']);
@@ -88,21 +85,21 @@ export class ConsoleFormatter implements TestFormatter {
8885
if (data.model && data.model !== this.currentModel) {
8986
this.currentModel = data.model;
9087
// Model changes are now handled by section headers
91-
} else if (data.message && !this.options.quiet) {
92-
// Only show progress messages in verbose mode or if explicitly needed
93-
if (this.options.verbose) {
88+
} else if (data.message) {
89+
// Only show progress messages in debug mode
90+
if (this.options.debug) {
9491
console.log(data.message);
9592
}
9693
}
9794
}
9895

9996
private handleTestStart(_data: TestStartEvent['data']): void {
10097
// For now, we don't show individual test starts
101-
// Could add verbose mode later that shows "Running: test_name..."
98+
// Could add debug mode later that shows "Running: test_name..."
10299
}
103100

104101
private handleTestComplete(data: TestCompleteEvent['data']): void {
105-
const { name, passed, errors, prompt } = data;
102+
const { name, passed, errors, prompt, messages } = data;
106103

107104
if (passed) {
108105
console.log(`✅ ${name}: PASSED`);
@@ -119,6 +116,14 @@ export class ConsoleFormatter implements TestFormatter {
119116
});
120117
}
121118
}
119+
120+
// Show conversation in debug mode
121+
if (this.options.debug && messages && messages.length > 0) {
122+
console.log();
123+
const formattedLines = formatConversation(messages, prompt);
124+
formattedLines.forEach(line => console.log(line));
125+
console.log();
126+
}
122127
}
123128

124129
private handleSuiteComplete(data: SuiteCompleteEvent['data']): void {

src/testing/display/formatters/JsonFormatter.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export class JsonFormatter implements TestFormatter {
2020
flush(): void {
2121
// TODO: Implement JSON output
2222
// Could output to file or stdout depending on options
23-
if (this.options.verbose) {
23+
if (this.options.debug) {
2424
console.log('JSON formatter not yet implemented');
2525
}
2626
}

src/testing/display/formatters/JunitXmlFormatter.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ export class JunitXmlFormatter implements TestFormatter {
282282
writeFileSync(this.outputFile, xml, 'utf8');
283283

284284
// Validate the generated XML
285-
if (this.options.verbose) {
285+
if (this.options.debug) {
286286
const validation = validateJunitXmlContent(xml);
287287
if (!validation.valid) {
288288
console.warn(`JUnit XML validation errors: ${validation.errors.join(', ')}`);

src/testing/display/types.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ export interface TestCompleteEvent extends TestEvent {
3030
passed: boolean;
3131
errors: string[];
3232
prompt?: string;
33+
messages?: any[]; // CoreMessage[] from 'ai' package
3334
};
3435
}
3536

@@ -84,8 +85,7 @@ export interface TestFormatter {
8485

8586
export interface DisplayOptions {
8687
formatter?: string;
87-
quiet?: boolean;
88-
verbose?: boolean;
88+
debug?: boolean;
8989
junitXml?: string;
9090
version?: string;
9191
}
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/**
2+
* Utility functions for formatting conversation messages for console output
3+
*/
4+
5+
export interface ConversationMessage {
6+
role: string;
7+
content: any;
8+
toolInvocations?: any[];
9+
}
10+
11+
/**
12+
* Format a conversation array into human-readable console output
13+
*/
14+
export function formatConversation(messages: ConversationMessage[], prompt?: string): string[] {
15+
const output: string[] = [];
16+
17+
// Add horizontal line at the start
18+
output.push(formatText('─'.repeat(60)));
19+
20+
// Include initial prompt if provided
21+
if (prompt) {
22+
output.push(formatRole('user'));
23+
output.push(formatText(prompt));
24+
output.push('');
25+
}
26+
27+
messages.forEach(msg => {
28+
// Format role header
29+
const roleDisplay = formatRole(msg.role);
30+
output.push(roleDisplay);
31+
32+
// Format content
33+
const contentLines = formatContent(msg.content);
34+
output.push(...contentLines);
35+
36+
// Handle legacy tool invocations
37+
if (msg.toolInvocations && msg.toolInvocations.length > 0) {
38+
msg.toolInvocations.forEach(tool => {
39+
const toolLine = formatToolCall(tool.toolName, tool.args);
40+
output.push(toolLine);
41+
});
42+
}
43+
44+
// Add space between messages
45+
output.push('');
46+
});
47+
48+
// Add horizontal line at the end
49+
output.push(formatText('─'.repeat(60)));
50+
51+
return output;
52+
}
53+
54+
/**
55+
* Format role with appropriate colors
56+
*/
57+
export function formatRole(role: string): string {
58+
switch (role) {
59+
case 'user':
60+
return '\x1b[32m[USER]\x1b[0m'; // Green
61+
case 'assistant':
62+
return '\x1b[36m[ASSISTANT]\x1b[0m'; // Cyan
63+
case 'system':
64+
return '\x1b[33m[SYSTEM]\x1b[0m'; // Yellow
65+
case 'tool':
66+
return '\x1b[35m[TOOL]\x1b[0m'; // Magenta
67+
default:
68+
return `\x1b[37m[${role.toUpperCase()}]\x1b[0m`; // White
69+
}
70+
}
71+
72+
/**
73+
* Format text content with darker color
74+
*/
75+
export function formatText(text: string): string {
76+
return `\x1b[90m${text}\x1b[0m`; // Dark gray/dim
77+
}
78+
79+
/**
80+
* Format content based on type
81+
*/
82+
export function formatContent(content: any): string[] {
83+
const lines: string[] = [];
84+
85+
if (typeof content === 'string') {
86+
lines.push(formatText(content));
87+
} else if (Array.isArray(content)) {
88+
// Handle content array (multimodal messages)
89+
content.forEach((part: any) => {
90+
if (part.type === 'text') {
91+
lines.push(formatText(part.text));
92+
} else if (part.type === 'tool-call') {
93+
const toolLine = formatToolCall(part.toolName, part.args);
94+
lines.push(toolLine);
95+
} else if (part.type === 'tool-result') {
96+
const resultLine = formatToolResult(part.result);
97+
lines.push(resultLine);
98+
}
99+
});
100+
} else {
101+
// Fallback for other content types
102+
lines.push(formatText(JSON.stringify(content)));
103+
}
104+
105+
return lines;
106+
}
107+
108+
/**
109+
* Format tool call
110+
*/
111+
export function formatToolCall(toolName: string, args: any): string {
112+
const argsStr = JSON.stringify(args);
113+
return formatText(`🔧 Tool Call: ${toolName}(${argsStr})`);
114+
}
115+
116+
/**
117+
* Format tool result
118+
*/
119+
export function formatToolResult(result: any): string {
120+
const resultStr = JSON.stringify(result);
121+
const truncated = resultStr.length > 200 ? `${resultStr.substring(0, 200)}...` : resultStr;
122+
return formatText(`✅ Tool Result: ${truncated}`);
123+
}

src/testing/evals/runner.ts

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ export interface EvalSummary {
1919
interface EvalServerOptions {
2020
serverConfig: ServerConfig;
2121
timeout?: number;
22-
quiet?: boolean;
23-
verbose?: boolean;
22+
debug?: boolean;
2423
}
2524

2625
export class EvalTestRunner {
@@ -92,7 +91,8 @@ export class EvalTestRunner {
9291
result.passed,
9392
result.errors,
9493
model,
95-
test.prompt
94+
test.prompt,
95+
result.messages
9696
);
9797
}
9898
}
@@ -109,15 +109,7 @@ export class EvalTestRunner {
109109
results,
110110
};
111111

112-
// Notify display manager about test suite completion
113-
if (this.displayManager) {
114-
this.displayManager.suiteComplete(
115-
summary.total,
116-
summary.passed,
117-
summary.failed,
118-
summary.duration
119-
);
120-
}
112+
// Note: suiteComplete is called by the main runner, not here
121113

122114
return summary;
123115
} finally {
@@ -212,7 +204,9 @@ export class EvalTestRunner {
212204
if (expectedToolCalls.required) {
213205
for (const requiredTool of expectedToolCalls.required) {
214206
if (!actualToolNames.includes(requiredTool)) {
215-
errors.push(`Required tool '${requiredTool}' was not called`);
207+
errors.push(
208+
`Required tool '${requiredTool}' was not called (actual calls: ${actualToolNames.length > 0 ? actualToolNames.join(', ') : 'none'})`
209+
);
216210
}
217211
}
218212
}
@@ -229,7 +223,9 @@ export class EvalTestRunner {
229223

230224
for (const actualTool of actualToolNames) {
231225
if (!allowedTools.includes(actualTool)) {
232-
errors.push(`Tool '${actualTool}' was called but not in allowed list`);
226+
errors.push(
227+
`Tool '${actualTool}' was called but not in allowed list (allowed: ${allowedTools.join(', ')})`
228+
);
233229
}
234230
}
235231
}

src/testing/runner.ts

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@ interface ServerOptions {
1616
serverConfig: string;
1717
serverName?: string;
1818
timeout?: number;
19-
quiet?: boolean;
20-
verbose?: boolean;
19+
debug?: boolean;
2120
junitXml?: string;
2221
}
2322

@@ -33,8 +32,7 @@ export class TestRunner {
3332
// Create display manager with options
3433
const displayOptions: DisplayOptions = {
3534
formatter: 'console',
36-
quiet: serverOptions.quiet,
37-
verbose: serverOptions.verbose,
35+
debug: serverOptions.debug,
3836
junitXml: serverOptions.junitXml,
3937
version: this.getVersion(),
4038
};
@@ -82,8 +80,7 @@ export class TestRunner {
8280
{
8381
serverConfig,
8482
timeout: this.serverOptions.timeout,
85-
quiet: this.serverOptions.quiet,
86-
verbose: this.serverOptions.verbose,
83+
debug: this.serverOptions.debug,
8784
},
8885
this.displayManager
8986
);
@@ -110,8 +107,7 @@ export class TestRunner {
110107
{
111108
serverConfig,
112109
timeout: this.serverOptions.timeout,
113-
quiet: this.serverOptions.quiet,
114-
verbose: this.serverOptions.verbose,
110+
debug: this.serverOptions.debug,
115111
},
116112
this.displayManager
117113
);

0 commit comments

Comments
 (0)