Skip to content

Commit 06a9591

Browse files
nmogil-twnmogil-tw
committed
updated per comments
1 parent eab4e4e commit 06a9591

File tree

1 file changed

+74
-71
lines changed

1 file changed

+74
-71
lines changed

‎src/cli/metrics/metrics-calculator.js‎

Lines changed: 74 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
const logger = require('../../utils/logger');
22

3+
const DEFAULT_METRICS = {
4+
tokensIn: 0,
5+
tokensOut: 0,
6+
totalCost: 0,
7+
cacheWrites: 0,
8+
cacheReads: 0,
9+
conversationHistoryIndex: 0
10+
};
11+
312
/**
413
* Calculator for processing and extracting metrics from task segments
514
*/
@@ -138,19 +147,17 @@ class MetricsCalculator {
138147
return false;
139148
}
140149

141-
for (const apiCall of this.segment.apiCalls) {
150+
return this.segment.apiCalls.some(apiCall => {
142151
if (apiCall.role === 'assistant' && apiCall.content && Array.isArray(apiCall.content)) {
143-
for (const content of apiCall.content) {
144-
if (content.type === 'text' && content.text &&
145-
(content.text.includes('use_mcp_tool') ||
146-
content.text.includes('use_mcp_server') ||
147-
content.text.includes('access_mcp_resource'))) {
148-
return true;
149-
}
150-
}
152+
return apiCall.content.some(content =>
153+
content.type === 'text' && content.text &&
154+
(content.text.includes('use_mcp_tool') ||
155+
content.text.includes('use_mcp_server') ||
156+
content.text.includes('access_mcp_resource'))
157+
);
151158
}
152-
}
153-
return false;
159+
return false;
160+
});
154161
}
155162

156163
/**
@@ -165,15 +172,13 @@ class MetricsCalculator {
165172
let apiCallCount = 0;
166173

167174
// Count API calls from UI messages only - only counting api_req_started events
168-
for (const msg of this.segment.userMessages) {
175+
this.segment.userMessages.forEach(msg => {
169176
// Count API request operations
170177
if (msg.type === 'say' && msg.say === 'api_req_started') {
171178
apiCallCount++;
172179
logger.debug(`Found API request in task ${this.segment.taskNumber}: ${msg.text ?? '[no text]'}`);
173180
}
174-
175-
// No longer counting MCP tool requests as per requirements
176-
}
181+
});
177182

178183
logger.info(`API Call Count for Task ${this.segment.taskNumber}: ${apiCallCount}`);
179184

@@ -201,39 +206,41 @@ class MetricsCalculator {
201206
let foundInitialTask = false;
202207

203208
// Look in API conversation history (most reliable source)
204-
for (const entry of this.segment.apiCalls) {
209+
this.segment.apiCalls.forEach(entry => {
205210
// Only process user messages
206-
if (entry.role === 'user' && entry.content && Array.isArray(entry.content)) {
207-
// Extract the text from all content items
208-
const fullText = entry.content
209-
.filter(item => item.type === 'text')
210-
.map(item => item.text ?? '')
211-
.join(' ');
212-
213-
// Check if this is the initial task request
214-
if (!foundInitialTask && (
215-
fullText.includes('Complete Task') ||
216-
fullText.includes('agent-instructions/mcp_instructions.md') ||
217-
fullText.includes('agent-instructions/control_instructions.md')
218-
)) {
219-
foundInitialTask = true;
220-
userInteractionCount = 1; // Set to exactly 1 for the initial task
221-
logger.info(`Found initial task instruction for task ${this.segment.taskNumber}`);
222-
continue; // Skip to next message
223-
}
224-
225-
// Only count additional user messages if they appear to be actual human follow-ups
226-
// and not system messages
227-
if (foundInitialTask &&
228-
!fullText.includes('<environment_details>') &&
229-
!fullText.startsWith('[') &&
230-
fullText.trim().length > 10) { // Minimum length to exclude noise
231-
// This appears to be a genuine follow-up question
232-
userInteractionCount++;
233-
logger.info(`Found follow-up user message for task ${this.segment.taskNumber}: "${fullText.substring(0, 50)}..."`);
234-
}
211+
if (entry.role !== 'user' || !entry.content || !Array.isArray(entry.content)) {
212+
return;
235213
}
236-
}
214+
215+
// Extract the text from all content items
216+
const fullText = entry.content
217+
.filter(item => item.type === 'text')
218+
.map(item => item.text ?? '')
219+
.join(' ');
220+
221+
// Check if this is the initial task request
222+
if (!foundInitialTask && (
223+
fullText.includes('Complete Task') ||
224+
fullText.includes('agent-instructions/mcp_instructions.md') ||
225+
fullText.includes('agent-instructions/control_instructions.md')
226+
)) {
227+
foundInitialTask = true;
228+
userInteractionCount = 1; // Set to exactly 1 for the initial task
229+
logger.info(`Found initial task instruction for task ${this.segment.taskNumber}`);
230+
return; // Skip to next message
231+
}
232+
233+
// Only count additional user messages if they appear to be actual human follow-ups
234+
// and not system messages
235+
if (foundInitialTask &&
236+
!fullText.includes('<environment_details>') &&
237+
!fullText.startsWith('[') &&
238+
fullText.trim().length > 10) { // Minimum length to exclude noise
239+
// This appears to be a genuine follow-up question
240+
userInteractionCount++;
241+
logger.info(`Found follow-up user message for task ${this.segment.taskNumber}: "${fullText.substring(0, 50)}..."`);
242+
}
243+
});
237244

238245
// If we still haven't found any interactions, default to 1
239246
if (userInteractionCount === 0) {
@@ -250,17 +257,10 @@ class MetricsCalculator {
250257
*/
251258
async calculateTokenMetrics() {
252259
// Default values
253-
const defaultMetrics = {
254-
tokensIn: 0,
255-
tokensOut: 0,
256-
totalCost: 0,
257-
cacheWrites: 0,
258-
cacheReads: 0,
259-
conversationHistoryIndex: 0
260-
};
260+
const metrics = { ...DEFAULT_METRICS };
261261

262262
if (!this.segment?.userMessages?.length) {
263-
return defaultMetrics;
263+
return metrics;
264264
}
265265

266266
let tokensIn = 0;
@@ -274,7 +274,7 @@ class MetricsCalculator {
274274
logger.info(`Starting token calculation for task ${this.segment.taskNumber}`);
275275

276276
// First pass: Collect reported token usage from Claude
277-
for (const message of this.segment.userMessages) {
277+
this.segment.userMessages.forEach(message => {
278278
if (message.type === 'say' && message.text) {
279279
try {
280280
const data = JSON.parse(message.text);
@@ -321,20 +321,20 @@ class MetricsCalculator {
321321
highestConvHistoryIndex = indexValue;
322322
}
323323
}
324-
}
324+
});
325325

326326
// Second pass: Check API calls for any additional token usage
327327
if (this.segment.apiCalls?.length) {
328328
let apiCallsWithTokens = 0;
329-
for (const apiCall of this.segment.apiCalls) {
329+
this.segment.apiCalls.forEach(apiCall => {
330330
if (apiCall.usage) {
331331
if (apiCall.usage.input_tokens) tokensIn += apiCall.usage.input_tokens;
332332
if (apiCall.usage.output_tokens) tokensOut += apiCall.usage.output_tokens;
333333
if (apiCall.usage.cost) totalCost += apiCall.usage.cost;
334334
apiCallsWithTokens++;
335335
logger.info(`API Call ${apiCallsWithTokens} tokens - In: ${apiCall.usage.input_tokens ?? 0}, Out: ${apiCall.usage.output_tokens ?? 0}`);
336336
}
337-
}
337+
});
338338
}
339339

340340
// Calculate total cost if not already set
@@ -381,24 +381,27 @@ class MetricsCalculator {
381381
}
382382

383383
// Try to determine from logs
384-
for (const apiCall of this.segment.apiCalls) {
384+
const modelFromLogs = this.segment.apiCalls.find(apiCall => {
385385
if (apiCall.role === 'assistant' && apiCall.content && Array.isArray(apiCall.content)) {
386-
for (const content of apiCall.content) {
386+
return apiCall.content.some(content => {
387387
if (content.type === 'text' && content.text) {
388388
// Look for model in system prompt
389389
const systemPromptMatch = content.text.match(/You are a powerful agentic AI coding assistant, powered by (Claude [\d.]+ \w+)/i);
390-
if (systemPromptMatch?.[1]) {
391-
// Normalize model name
392-
const detectedModel = systemPromptMatch[1].toLowerCase()
393-
.replace('claude ', 'claude-')
394-
.replace(' ', '-');
395-
396-
logger.info(`Detected model from logs: ${detectedModel}`);
397-
return detectedModel;
398-
}
390+
return systemPromptMatch?.[1];
399391
}
400-
}
392+
return false;
393+
});
401394
}
395+
return false;
396+
});
397+
398+
if (modelFromLogs) {
399+
const detectedModel = modelFromLogs.toLowerCase()
400+
.replace('claude ', 'claude-')
401+
.replace(' ', '-');
402+
403+
logger.info(`Detected model from logs: ${detectedModel}`);
404+
return detectedModel;
402405
}
403406

404407
// Default if not found in logs and no argument provided

0 commit comments

Comments
 (0)