diff --git a/src/extension/prompt/node/chatParticipantTelemetry.ts b/src/extension/prompt/node/chatParticipantTelemetry.ts index 1d230ae47..c0ceab602 100644 --- a/src/extension/prompt/node/chatParticipantTelemetry.ts +++ b/src/extension/prompt/node/chatParticipantTelemetry.ts @@ -25,6 +25,7 @@ import { Conversation } from '../common/conversation'; import { IToolCall, IToolCallRound } from '../common/intents'; import { IDocumentContext } from './documentContext'; import { IIntent, TelemetryData } from './intents'; +import { RepoInfoTelemetry } from './repoInfoTelemetry'; import { ConversationalBaseTelemetryData, createTelemetryWithId, extendUserMessageTelemetryData, getCodeBlocks, sendModelMessageTelemetry, sendOffTopicMessageTelemetry, sendUserActionTelemetry, sendUserMessageTelemetry } from './telemetry'; // #region: internal telemetry for responses @@ -200,6 +201,8 @@ export class ChatTelemetryBuilder { public readonly baseUserTelemetry: ConversationalBaseTelemetryData = createTelemetryWithId(); + private readonly _repoInfoTelemetry: RepoInfoTelemetry; + public get telemetryMessageId() { return this.baseUserTelemetry.properties.messageId; } @@ -211,9 +214,16 @@ export class ChatTelemetryBuilder { private readonly _firstTurn: boolean, private readonly _request: vscode.ChatRequest, @IInstantiationService private readonly instantiationService: IInstantiationService, - ) { } + ) { + // IANHU: Remove log later + console.log('repoInfoTelemetry created for messageId', this.baseUserTelemetry.properties.messageId); + this._repoInfoTelemetry = this.instantiationService.createInstance(RepoInfoTelemetry, this.baseUserTelemetry.properties.messageId); + } public makeRequest(intent: IIntent, location: ChatLocation, conversation: Conversation, messages: Raw.ChatMessage[], promptTokenLength: number, references: readonly PromptReference[], endpoint: IChatEndpoint, telemetryData: readonly TelemetryData[], availableToolCount: number): InlineChatTelemetry | PanelChatTelemetry { + this._repoInfoTelemetry.sendBeginTelemetryIfNeeded().catch(() => { + // Error logged in RepoInfoTelemetry + }); if (location === ChatLocation.Editor) { return this.instantiationService.createInstance(InlineChatTelemetry, @@ -231,6 +241,8 @@ export class ChatTelemetryBuilder { promptTokenLength, telemetryData, availableToolCount, + // IANHU: Don't send to inline? + this._repoInfoTelemetry ); } else { return this.instantiationService.createInstance(PanelChatTelemetry, @@ -248,6 +260,7 @@ export class ChatTelemetryBuilder { promptTokenLength, telemetryData, availableToolCount, + this._repoInfoTelemetry ); } } @@ -298,6 +311,7 @@ export abstract class ChatTelemetry tool.name)) }, toolCallMeasurements); + this._repoInfoTelemetry.sendEndTelemetry().catch(() => { + // Error logged in RepoInfoTelemetry + }); } protected abstract _sendInternalRequestTelemetryEvent(): void; @@ -492,7 +509,6 @@ export abstract class ChatTelemetry(ctor: new (...args: any[]) => T): T | undefined { return this._genericTelemetryData.find(d => d instanceof ctor); } - } export class PanelChatTelemetry extends ChatTelemetry { @@ -512,6 +528,7 @@ export class PanelChatTelemetry extends ChatTelemetry { promptTokenLength: number, genericTelemetryData: readonly TelemetryData[], availableToolCount: number, + repoInfoTelemetry: RepoInfoTelemetry, @ITelemetryService telemetryService: ITelemetryService, @ILanguageDiagnosticsService private readonly _languageDiagnosticsService: ILanguageDiagnosticsService, ) { @@ -763,6 +780,7 @@ export class InlineChatTelemetry extends ChatTelemetry { promptTokenLength, genericTelemetryData, availableToolCount, + repoInfoTelemetry, telemetryService ); diff --git a/src/extension/prompt/node/repoInfoTelemetry.ts b/src/extension/prompt/node/repoInfoTelemetry.ts new file mode 100644 index 000000000..9f4e8705b --- /dev/null +++ b/src/extension/prompt/node/repoInfoTelemetry.ts @@ -0,0 +1,193 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ICopilotTokenStore } from '../../../platform/authentication/common/copilotTokenStore'; +import { IFileSystemService } from '../../../platform/filesystem/common/fileSystemService'; +import { IGitDiffService } from '../../../platform/git/common/gitDiffService'; +import { IGitExtensionService } from '../../../platform/git/common/gitExtensionService'; +import { getGitHubRepoInfoFromContext, IGitService } from '../../../platform/git/common/gitService'; +import { ILogService } from '../../../platform/log/common/logService'; +import { ITelemetryService, multiplexProperties, wouldMultiplexTelemetryPropertyBeTruncated } from '../../../platform/telemetry/common/telemetry'; + +// EVENT: repoInfo +type RepoInfoTelemetryProperties = { + remoteUrl: string | undefined; + headCommitHash: string | undefined; + diffsJSON: string | undefined; + result: 'success' | 'filesChanged' | 'diffTooLarge'; +}; + +type RepoInfoInternalTelemetryProperties = RepoInfoTelemetryProperties & { + location: 'begin' | 'end'; + telemetryMessageId: string; +}; + +export class RepoInfoTelemetry { + private _beginTelemetrySent = false; + private _beginTelemetryPromise: Promise | undefined; + + constructor( + private readonly _telemetryMessageId: string, + @ITelemetryService private readonly _telemetryService: ITelemetryService, + @IGitService private readonly _gitService: IGitService, + @IGitDiffService private readonly _gitDiffService: IGitDiffService, + @IGitExtensionService private readonly _gitExtensionService: IGitExtensionService, + @ICopilotTokenStore private readonly _copilotTokenStore: ICopilotTokenStore, + @ILogService private readonly _logService: ILogService, + @IFileSystemService private readonly _fileSystemService: IFileSystemService, + ) { } + + public async sendBeginTelemetryIfNeeded(): Promise { + // IANHU: Remove + console.log('repoInfoTelemetry: sendBeginTelemetryIfNeeded called'); + + if (this._beginTelemetrySent) { + // Already sent or in progress + return this._beginTelemetryPromise; + } + + this._beginTelemetrySent = true; + this._beginTelemetryPromise = this._sendRepoInfoTelemetry('begin'); + + return this._beginTelemetryPromise.catch((error) => { + this._logService.warn(`Failed to send begin repo info telemetry ${error}`); + }); + } + + public async sendEndTelemetry(): Promise { + // IANHU: Remove + console.log('repoInfoTelemetry: sendEndTelemetry called'); + + await this._beginTelemetryPromise; + + return this._sendRepoInfoTelemetry('end').catch((error) => { + this._logService.warn(`Failed to send end repo info telemetry ${error}`); + }); + } + + private async _sendRepoInfoTelemetry(location: 'begin' | 'end'): Promise { + // IANHU: Remove + console.log('repoInfoTelemetry: sendRepoInfoTelemetry called', location); + + if (this._copilotTokenStore.copilotToken?.isInternal !== true) { + return; + } + + const gitInfo = await this._getRepoInfoTelemetry(); + if (!gitInfo) { + return; + } + + const properties = multiplexProperties({ + ...gitInfo, + location, + telemetryMessageId: this._telemetryMessageId + } as RepoInfoInternalTelemetryProperties); + + this._telemetryService.sendInternalMSFTTelemetryEvent('request.repoInfo', properties); + + + // IANHU: Remove logging later + console.log(JSON.stringify({ + name: 'request.repoInfo', + data: { + ...gitInfo, + location, + telemetryMessageId: this._telemetryMessageId + } + })); + } + + private async _getRepoInfoTelemetry(): Promise { + const repoContext = this._gitService.activeRepository.get(); + + if (!repoContext || !repoContext.changes) { + return; + } + + const githubInfo = getGitHubRepoInfoFromContext(repoContext); + if (!githubInfo) { + return; + } + + // Get the upstream commit from the repository + const gitAPI = this._gitExtensionService.getExtensionApi(); + const repository = gitAPI?.getRepository(repoContext.rootUri); + const upstreamCommit = repository?.state.HEAD?.upstream?.commit; + if (!upstreamCommit) { + return; + } + + // Before we calculate our async diffs, sign up for file system change events + // Any changes during the async operations will invalidate our diff data and we send it + // as a failure without a diffs + const watcher = this._fileSystemService.createFileSystemWatcher('**/*'); + let filesChanged = false; + const createDisposable = watcher.onDidCreate(() => filesChanged = true); + const changeDisposable = watcher.onDidChange(() => filesChanged = true); + const deleteDisposable = watcher.onDidDelete(() => filesChanged = true); + + try { + const changes = await this._gitService.diffWith(repoContext.rootUri, '@{upstream}'); + if (!changes || changes.length === 0) { + return; + } + + // Check if files changed during the git diff operation + if (filesChanged) { + return { + remoteUrl: githubInfo.remoteUrl, + headCommitHash: upstreamCommit, + diffsJSON: undefined, + result: 'filesChanged', + }; + } + + const diffs = (await this._gitDiffService.getChangeDiffs(repoContext.rootUri, changes)).map(diff => { + return { + uri: diff.uri.toString(), + originalUri: diff.originalUri.toString(), + renameUri: diff.renameUri?.toString(), + status: diff.status, + diff: diff.diff, + }; + }); + + // Final check if files changed during the diff processing + if (filesChanged) { + return { + remoteUrl: githubInfo.remoteUrl, + headCommitHash: upstreamCommit, + diffsJSON: undefined, + result: 'filesChanged', + }; + } + + const diffsJSON = diffs.length > 0 ? JSON.stringify(diffs) : undefined; + + // Check if the diff is too big and notify that + if (wouldMultiplexTelemetryPropertyBeTruncated(diffsJSON)) { + return { + remoteUrl: githubInfo.remoteUrl, + headCommitHash: upstreamCommit, + diffsJSON: undefined, + result: 'diffTooLarge', + }; + } + + return { + remoteUrl: githubInfo.remoteUrl, + headCommitHash: upstreamCommit, + diffsJSON, + result: 'success', + }; + } finally { + createDisposable.dispose(); + changeDisposable.dispose(); + deleteDisposable.dispose(); + watcher.dispose(); + } + } +} \ No newline at end of file diff --git a/src/platform/telemetry/common/telemetry.ts b/src/platform/telemetry/common/telemetry.ts index 5782b2cc9..978a98868 100644 --- a/src/platform/telemetry/common/telemetry.ts +++ b/src/platform/telemetry/common/telemetry.ts @@ -214,4 +214,8 @@ export function multiplexProperties(properties: { [key: string]: string | undefi } } return newProperties; +} + +export function wouldMultiplexTelemetryPropertyBeTruncated(propertyValue: string | undefined): boolean { + return (propertyValue?.length ?? 0) > (MAX_PROPERTY_LENGTH * MAX_CONCATENATED_PROPERTIES); } \ No newline at end of file diff --git a/src/util/common/chatResponseStreamImpl.ts b/src/util/common/chatResponseStreamImpl.ts index b3445ae92..faadeef98 100644 --- a/src/util/common/chatResponseStreamImpl.ts +++ b/src/util/common/chatResponseStreamImpl.ts @@ -124,6 +124,8 @@ export class ChatResponseStreamImpl implements FinalizableChatResponseStream { } textEdit(target: Uri, editsOrDone: TextEdit | TextEdit[] | true): void { + // IANHU: Just debug logging + console.log(`repoInfo textedit ${target.fsPath}`); if (Array.isArray(editsOrDone) || editsOrDone instanceof TextEdit) { this._push(new ChatResponseTextEditPart(target, editsOrDone)); } else {