Skip to content

Commit cc2e907

Browse files
yasithrashanyasithrashan
committed
Refactor project metrics calculation and add feedback telemetry
1 parent fe3776b commit cc2e907

File tree

6 files changed

+164
-63
lines changed

6 files changed

+164
-63
lines changed

‎workspaces/ballerina/ballerina-extension/src/features/ai/service/design/design.ts‎

Lines changed: 14 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import { getProjectSource } from "../../utils/project-utils";
4141
import { sendTelemetryEvent, sendTelemetryException } from "../../../telemetry";
4242
import { TM_EVENT_BALLERINA_AI_GENERATION_SUBMITTED, TM_EVENT_BALLERINA_AI_GENERATION_COMPLETED, TM_EVENT_BALLERINA_AI_GENERATION_FAILED, TM_EVENT_BALLERINA_AI_GENERATION_ABORTED, CMP_BALLERINA_AI_GENERATION } from "../../../telemetry";
4343
import { extension } from "../../../../BalExtensionContext";
44+
import { getProjectMetrics } from "../../../telemetry/common/project-metrics";
4445

4546

4647
const LANGFUSE_SECRET = process.env.LANGFUSE_SECRET;
@@ -78,32 +79,8 @@ export async function generateDesignCore(
7879

7980
const cacheOptions = await getProviderCacheControl();
8081

81-
// Get state machine context for telemetry
8282
const stateContext = AIChatStateMachine.context();
83-
84-
let totalFileCount = 0;
85-
let totalLineCount = 0;
86-
87-
for (const project of projects) {
88-
const projectFiles = project.sourceFiles || [];
89-
totalFileCount += projectFiles.length;
90-
91-
for (const file of projectFiles) {
92-
totalLineCount += file.content.split('\n').length;
93-
}
94-
95-
// Also count module files if present
96-
if (project.projectModules) {
97-
for (const module of project.projectModules) {
98-
const moduleFiles = module.sourceFiles || [];
99-
totalFileCount += moduleFiles.length;
100-
101-
for (const file of moduleFiles) {
102-
totalLineCount += file.content.split('\n').length;
103-
}
104-
}
105-
}
106-
}
83+
const projectMetrics = await getProjectMetrics();
10784

10885
// Send telemetry when the user submits a query
10986
sendTelemetryEvent(extension.ballerinaExtInstance, TM_EVENT_BALLERINA_AI_GENERATION_SUBMITTED, CMP_BALLERINA_AI_GENERATION, {
@@ -113,8 +90,8 @@ export async function generateDesignCore(
11390
operationType: params.operationType,
11491
isPlanMode: isPlanModeEnabled.toString(),
11592
approvalMode: stateContext.autoApproveEnabled ? 'auto' : 'manual',
116-
inputFileCount: totalFileCount.toString(),
117-
inputLineCount: totalLineCount.toString(),
93+
inputFileCount: projectMetrics.fileCount.toString(),
94+
inputLineCount: projectMetrics.lineCount.toString(),
11895
});
11996

12097
const modifiedFiles: string[] = [];
@@ -158,7 +135,7 @@ export async function generateDesignCore(
158135
let diagnosticCheckCount = 0;
159136
let totalCompilationErrorsDuringGeneration = 0;
160137

161-
const { fullStream, response, usage: usagePromise } = streamText({
138+
const { fullStream, response, usage } = streamText({
162139
model: await getAnthropicClient(ANTHROPIC_SONNET_4),
163140
maxOutputTokens: 8192,
164141
temperature: 0,
@@ -366,11 +343,10 @@ Generation stopped by user. The last in-progress task was not saved. Files have
366343
const assistantMessages = finalResponse.messages || [];
367344
const generationEndTime = Date.now();
368345

369-
// Extract token usage information
370-
const usage = await usagePromise;
371-
const inputTokens = usage.inputTokens || 0;
372-
const outputTokens = usage.outputTokens || 0;
373-
const totalTokens = usage.totalTokens || 0;
346+
const usageInfo = await usage;
347+
const inputTokens = usageInfo.inputTokens || 0;
348+
const outputTokens = usageInfo.outputTokens || 0;
349+
const totalTokens = usageInfo.totalTokens || 0;
374350

375351
const finalDiagnostics = await checkCompilationErrors(tempProjectPath);
376352
if (finalDiagnostics.diagnostics && finalDiagnostics.diagnostics.length > 0) {
@@ -392,6 +368,9 @@ Generation stopped by user. The last in-progress task was not saved. Files have
392368
updateAndSaveChat(messageId, userMessageContent, assistantMessages, eventHandler);
393369
eventHandler({ type: "stop", command: Command.Design });
394370

371+
// Get final project metrics after generation
372+
const finalProjectMetrics = await getProjectMetrics();
373+
395374
// Send telemetry for generation completion
396375
sendTelemetryEvent(extension.ballerinaExtInstance, TM_EVENT_BALLERINA_AI_GENERATION_COMPLETED, CMP_BALLERINA_AI_GENERATION, {
397376
projectId: stateContext.projectId || 'unknown',
@@ -408,6 +387,8 @@ Generation stopped by user. The last in-progress task was not saved. Files have
408387
inputTokens: inputTokens.toString(),
409388
outputTokens: outputTokens.toString(),
410389
totalTokens: totalTokens.toString(),
390+
outputFileCount: finalProjectMetrics.fileCount.toString(),
391+
outputLineCount: finalProjectMetrics.lineCount.toString(),
411392
});
412393

413394
AIChatStateMachine.sendEvent({

‎workspaces/ballerina/ballerina-extension/src/features/ai/service/feedback/feedback.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com/) All Rights Reserved.
2+
3+
// WSO2 LLC. licenses this file to you under the Apache License,
4+
// Version 2.0 (the "License"); you may not use this file except
5+
// in compliance with the License.
6+
// You may obtain a copy of the License at
7+
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
10+
// Unless required by applicable law or agreed to in writing,
11+
// software distributed under the License is distributed on an
12+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13+
// KIND, either express or implied. See the License for the
14+
// specific language governing permissions and limitations
15+
// under the License.
16+
17+
import { SubmitFeedbackRequest } from "@wso2/ballerina-core";
18+
import { fetchWithAuth } from "../connection";
19+
import { OLD_BACKEND_URL } from "../../utils";
20+
import { extension } from "../../../../BalExtensionContext";
21+
import { sendTelemetryEvent, TM_EVENT_BALLERINA_AI_GENERATION_FEEDBACK, CMP_BALLERINA_AI_GENERATION } from "../../../telemetry";
22+
import { cleanDiagnosticMessages } from "../../../../rpc-managers/ai-panel/utils";
23+
24+
export async function submitFeedback(content: SubmitFeedbackRequest): Promise<boolean> {
25+
try {
26+
sendTelemetryEvent(
27+
extension.ballerinaExtInstance,
28+
TM_EVENT_BALLERINA_AI_GENERATION_FEEDBACK,
29+
CMP_BALLERINA_AI_GENERATION,
30+
{
31+
feedbackType: content.positive ? 'positive' : 'negative',
32+
hasFeedbackText: content.feedbackText ? 'true' : 'false',
33+
feedbackTextLength: content.feedbackText?.length.toString() || '0',
34+
}
35+
);
36+
37+
const payload = {
38+
feedback: content.feedbackText,
39+
positive: content.positive,
40+
messages: content.messages,
41+
diagnostics: cleanDiagnosticMessages(content.diagnostics)
42+
};
43+
44+
const response = await fetchWithAuth(`${OLD_BACKEND_URL}/feedback`, {
45+
method: 'POST',
46+
headers: {
47+
'Content-Type': 'application/json'
48+
},
49+
body: JSON.stringify(payload)
50+
});
51+
52+
if (response.ok) {
53+
return true;
54+
} else {
55+
console.error("Failed to submit feedback");
56+
return false;
57+
}
58+
} catch (error) {
59+
console.error("Error submitting feedback:", error);
60+
return false;
61+
}
62+
}

‎workspaces/ballerina/ballerina-extension/src/features/telemetry/common/project-metrics.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com/) All Rights Reserved.
2+
3+
// WSO2 LLC. licenses this file to you under the Apache License,
4+
// Version 2.0 (the "License"); you may not use this file except
5+
// in compliance with the License.
6+
// You may obtain a copy of the License at
7+
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
10+
// Unless required by applicable law or agreed to in writing,
11+
// software distributed under the License is distributed on an
12+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13+
// KIND, either express or implied. See the License for the
14+
// specific language governing permissions and limitations
15+
// under the License.
16+
17+
import * as vscode from 'vscode';
18+
import * as fs from 'fs';
19+
20+
export interface ProjectMetrics {
21+
fileCount: number;
22+
lineCount: number;
23+
}
24+
25+
export async function getProjectMetrics(): Promise<ProjectMetrics> {
26+
const workspaceFolders = vscode.workspace.workspaceFolders;
27+
28+
if (!workspaceFolders || workspaceFolders.length === 0) {
29+
return { fileCount: 0, lineCount: 0 };
30+
}
31+
const files = await vscode.workspace.findFiles(
32+
'**/*.bal',
33+
'**/target/**'
34+
);
35+
36+
let totalFileCount = 0;
37+
let totalLineCount = 0;
38+
39+
for (const fileUri of files) {
40+
try {
41+
totalFileCount++;
42+
const fileContent = await fs.promises.readFile(fileUri.fsPath, 'utf8');
43+
const lineCount = fileContent.split('\n').length;
44+
totalLineCount += lineCount;
45+
} catch (error) {
46+
console.warn(`Failed to read file ${fileUri.fsPath}:`, error);
47+
}
48+
}
49+
50+
return {
51+
fileCount: totalFileCount,
52+
lineCount: totalLineCount
53+
};
54+
}

‎workspaces/ballerina/ballerina-extension/src/features/telemetry/events.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,4 +127,5 @@ export const TM_EVENT_BALLERINA_AI_GENERATION_SUBMITTED = "ballerina.ai.generati
127127
export const TM_EVENT_BALLERINA_AI_GENERATION_COMPLETED = "ballerina.ai.generation.completed";
128128
export const TM_EVENT_BALLERINA_AI_GENERATION_FAILED = "ballerina.ai.generation.failed";
129129
export const TM_EVENT_BALLERINA_AI_GENERATION_ABORTED = "ballerina.ai.generation.aborted";
130-
export const TM_EVENT_BALLERINA_AI_GENERATION_REVERTED = "ballerina.ai.generation.reverted";
130+
export const TM_EVENT_BALLERINA_AI_GENERATION_REVERTED = "ballerina.ai.generation.reverted";
131+
export const TM_EVENT_BALLERINA_AI_GENERATION_FEEDBACK = "ballerina.ai.generation.feedback";

‎workspaces/ballerina/ballerina-extension/src/features/telemetry/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ import { BallerinaExtension } from "../../core";
2121
import { getLoginMethod, getBiIntelId } from "../../utils/ai/auth";
2222

2323
//Ballerina-VSCode-Extention repo key as default
24-
const DEFAULT_KEY = "3a82b093-5b7b-440c-9aa2-3b8e8e5704e7";
24+
// const DEFAULT_KEY = "3a82b093-5b7b-440c-9aa2-3b8e8e5704e7";
25+
const DEFAULT_KEY = "ff7807cf-db47-4e8b-bd70-03fdfc2576b7";
2526
const INSTRUMENTATION_KEY = process.env.CODE_SERVER_ENV && process.env.VSCODE_CHOREO_INSTRUMENTATION_KEY ? process.env.VSCODE_CHOREO_INSTRUMENTATION_KEY : DEFAULT_KEY;
2627
const isWSO2User = process.env.VSCODE_CHOREO_USER_EMAIL ? process.env.VSCODE_CHOREO_USER_EMAIL.endsWith('@wso2.com') : false;
2728
const isAnonymous = process.env.VSCODE_CHOREO_USER_EMAIL ? process.env.VSCODE_CHOREO_USER_EMAIL.endsWith('@choreo.dev') : false;

‎workspaces/ballerina/ballerina-extension/src/rpc-managers/ai-panel/rpc-manager.ts‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ import { AIChatMachineEventType } from "@wso2/ballerina-core/lib/state-machine-t
8080
import { checkToken } from "../../../src/views/ai-panel/utils";
8181
import { extension } from "../../BalExtensionContext";
8282
import { generateDocumentationForService } from "../../features/ai/service/documentation/doc_generator";
83+
import { submitFeedback as submitFeedbackService } from "../../features/ai/service/feedback/feedback";
8384
// import { generateHealthcareCode } from "../../features/ai/service/healthcare/healthcare";
8485
import { selectRequiredFunctions } from "../../features/ai/service/libs/funcs";
8586
import { GenerationType } from "../../features/ai/service/libs/libs";
@@ -595,34 +596,35 @@ export class AiPanelRpcManager implements AIPanelAPI {
595596
}
596597

597598
async submitFeedback(content: SubmitFeedbackRequest): Promise<boolean> {
598-
return new Promise(async (resolve) => {
599-
try {
600-
const payload = {
601-
feedback: content.feedbackText,
602-
positive: content.positive,
603-
messages: content.messages,
604-
diagnostics: cleanDiagnosticMessages(content.diagnostics)
605-
};
606-
607-
const response = await fetchWithAuth(`${OLD_BACKEND_URL}/feedback`, {
608-
method: 'POST',
609-
headers: {
610-
'Content-Type': 'application/json'
611-
},
612-
body: JSON.stringify(payload)
613-
});
614-
615-
if (response.ok) {
616-
resolve(true);
617-
} else {
618-
console.error("Failed to submit feedback");
619-
resolve(false);
620-
}
621-
} catch (error) {
622-
console.error("Error submitting feedback:", error);
623-
resolve(false);
624-
}
625-
});
599+
return submitFeedbackService(content);
600+
// return new Promise(async (resolve) => {
601+
// try {
602+
// const payload = {
603+
// feedback: content.feedbackText,
604+
// positive: content.positive,
605+
// messages: content.messages,
606+
// diagnostics: cleanDiagnosticMessages(content.diagnostics)
607+
// };
608+
609+
// const response = await fetchWithAuth(`${OLD_BACKEND_URL}/feedback`, {
610+
// method: 'POST',
611+
// headers: {
612+
// 'Content-Type': 'application/json'
613+
// },
614+
// body: JSON.stringify(payload)
615+
// });
616+
617+
// if (response.ok) {
618+
// resolve(true);
619+
// } else {
620+
// console.error("Failed to submit feedback");
621+
// resolve(false);
622+
// }
623+
// } catch (error) {
624+
// console.error("Error submitting feedback:", error);
625+
// resolve(false);
626+
// }
627+
// });
626628
}
627629

628630
async getRelevantLibrariesAndFunctions(params: RelevantLibrariesAndFunctionsRequest): Promise<RelevantLibrariesAndFunctionsResponse> {

0 commit comments

Comments
 (0)