DRAFT (no review): New telemetry request from science team

src/extension/prompt/node/chatParticipantTelemetry.ts

@@ -25,6 +25,7 @@ import { Conversation } from '../common/conversation';
 import { IToolCall, IToolCallRound } from '../common/intents';
 import { IDocumentContext } from './documentContext';
 import { IIntent, TelemetryData } from './intents';
+import { RepoInfoTelemetry } from './repoInfoTelemetry';
 import { ConversationalBaseTelemetryData, createTelemetryWithId, extendUserMessageTelemetryData, getCodeBlocks, sendModelMessageTelemetry, sendOffTopicMessageTelemetry, sendUserActionTelemetry, sendUserMessageTelemetry } from './telemetry';
 
 // #region: internal telemetry for responses
@@ -200,6 +201,8 @@ export class ChatTelemetryBuilder {
 
 	public readonly baseUserTelemetry: ConversationalBaseTelemetryData = createTelemetryWithId();
 
+	private readonly _repoInfoTelemetry: RepoInfoTelemetry;
+
 	public get telemetryMessageId() {
 		return this.baseUserTelemetry.properties.messageId;
 	}
@@ -211,9 +214,16 @@ export class ChatTelemetryBuilder {
 		private readonly _firstTurn: boolean,
 		private readonly _request: vscode.ChatRequest,
 		@IInstantiationService private readonly instantiationService: IInstantiationService,
-	) { }
+	) {
+		// IANHU: Remove log later
+		console.log('repoInfoTelemetry created for messageId', this.baseUserTelemetry.properties.messageId);
+		this._repoInfoTelemetry = this.instantiationService.createInstance(RepoInfoTelemetry, this.baseUserTelemetry.properties.messageId);
+	}
 
 	public makeRequest(intent: IIntent, location: ChatLocation, conversation: Conversation, messages: Raw.ChatMessage[], promptTokenLength: number, references: readonly PromptReference[], endpoint: IChatEndpoint, telemetryData: readonly TelemetryData[], availableToolCount: number): InlineChatTelemetry | PanelChatTelemetry {
+		this._repoInfoTelemetry.sendBeginTelemetryIfNeeded().catch(() => {
+			// Error logged in RepoInfoTelemetry
+		});
 
 		if (location === ChatLocation.Editor) {
 			return this.instantiationService.createInstance(InlineChatTelemetry,
@@ -231,6 +241,8 @@ export class ChatTelemetryBuilder {
 				promptTokenLength,
 				telemetryData,
 				availableToolCount,
+				// IANHU: Don't send to inline?
+				this._repoInfoTelemetry
 			);
 		} else {
 			return this.instantiationService.createInstance(PanelChatTelemetry,
@@ -248,6 +260,7 @@ export class ChatTelemetryBuilder {
 				promptTokenLength,
 				telemetryData,
 				availableToolCount,
+				this._repoInfoTelemetry
 			);
 		}
 	}
@@ -298,6 +311,7 @@ export abstract class ChatTelemetry<C extends IDocumentContext | undefined = IDo
 		promptTokenLength: number,
 		protected readonly _genericTelemetryData: readonly TelemetryData[],
 		protected readonly _availableToolCount: number,
+		protected readonly _repoInfoTelemetry: RepoInfoTelemetry,
 		@ITelemetryService protected readonly _telemetryService: ITelemetryService,
 	) {
 		// Extend the base user telemetry with message and prompt information.
@@ -481,6 +495,9 @@ export abstract class ChatTelemetry<C extends IDocumentContext | undefined = IDo
 			availableTools: JSON.stringify(availableTools.map(tool => tool.name))
 		}, toolCallMeasurements);
 
+		this._repoInfoTelemetry.sendEndTelemetry().catch(() => {
+			// Error logged in RepoInfoTelemetry
+		});
 	}
 
 	protected abstract _sendInternalRequestTelemetryEvent(): void;
@@ -492,7 +509,6 @@ export abstract class ChatTelemetry<C extends IDocumentContext | undefined = IDo
 	protected _getTelemetryData<T extends TelemetryData>(ctor: new (...args: any[]) => T): T | undefined {
 		return <T>this._genericTelemetryData.find(d => d instanceof ctor);
 	}
-
 }
 
 export class PanelChatTelemetry extends ChatTelemetry<IDocumentContext | undefined> {
@@ -512,6 +528,7 @@ export class PanelChatTelemetry extends ChatTelemetry<IDocumentContext | undefin
 		promptTokenLength: number,
 		genericTelemetryData: readonly TelemetryData[],
 		availableToolCount: number,
+		repoInfoTelemetry: RepoInfoTelemetry,
 		@ITelemetryService telemetryService: ITelemetryService,
 		@IConfigurationService private readonly _configurationService: IConfigurationService,
 	) {
@@ -530,13 +547,12 @@ export class PanelChatTelemetry extends ChatTelemetry<IDocumentContext | undefin
 			promptTokenLength,
 			genericTelemetryData,
 			availableToolCount,
+			repoInfoTelemetry,
 			telemetryService
 		);
 	}
 
 	protected override _sendInternalRequestTelemetryEvent(): void {
-
-
 		// Capture the created prompt in internal telemetry
 		this._telemetryService.sendInternalMSFTTelemetryEvent('interactiveSessionMessage', {
 			chatLocation: 'panel',
@@ -745,6 +761,7 @@ export class InlineChatTelemetry extends ChatTelemetry<IDocumentContext> {
 		promptTokenLength: number,
 		genericTelemetryData: readonly TelemetryData[],
 		availableToolCount: number,
+		repoInfoTelemetry: RepoInfoTelemetry,
 		@ITelemetryService telemetryService: ITelemetryService,
 		@ILanguageDiagnosticsService private readonly _languageDiagnosticsService: ILanguageDiagnosticsService,
 	) {
@@ -763,6 +780,7 @@ export class InlineChatTelemetry extends ChatTelemetry<IDocumentContext> {
 			promptTokenLength,
 			genericTelemetryData,
 			availableToolCount,
+			repoInfoTelemetry,
 			telemetryService
 		);
 

src/extension/prompt/node/repoInfoTelemetry.ts

@@ -0,0 +1,193 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ICopilotTokenStore } from '../../../platform/authentication/common/copilotTokenStore';
+import { IFileSystemService } from '../../../platform/filesystem/common/fileSystemService';
+import { IGitDiffService } from '../../../platform/git/common/gitDiffService';
+import { IGitExtensionService } from '../../../platform/git/common/gitExtensionService';
+import { getGitHubRepoInfoFromContext, IGitService } from '../../../platform/git/common/gitService';
+import { ILogService } from '../../../platform/log/common/logService';
+import { ITelemetryService, multiplexProperties, wouldMultiplexTelemetryPropertyBeTruncated } from '../../../platform/telemetry/common/telemetry';
+
+// EVENT: repoInfo
+type RepoInfoTelemetryProperties = {
+	remoteUrl: string | undefined;
+	headCommitHash: string | undefined;
+	diffsJSON: string | undefined;
+	result: 'success' | 'filesChanged' | 'diffTooLarge';
+};
+
+type RepoInfoInternalTelemetryProperties = RepoInfoTelemetryProperties & {
+	location: 'begin' | 'end';
+	telemetryMessageId: string;
+};
+
+export class RepoInfoTelemetry {
+	private _beginTelemetrySent = false;
+	private _beginTelemetryPromise: Promise<void> | undefined;
+
+	constructor(
+		private readonly _telemetryMessageId: string,
+		@ITelemetryService private readonly _telemetryService: ITelemetryService,
+		@IGitService private readonly _gitService: IGitService,
+		@IGitDiffService private readonly _gitDiffService: IGitDiffService,
+		@IGitExtensionService private readonly _gitExtensionService: IGitExtensionService,
+		@ICopilotTokenStore private readonly _copilotTokenStore: ICopilotTokenStore,
+		@ILogService private readonly _logService: ILogService,
+		@IFileSystemService private readonly _fileSystemService: IFileSystemService,
+	) { }
+
+	public async sendBeginTelemetryIfNeeded(): Promise<void> {
+		// IANHU: Remove
+		console.log('repoInfoTelemetry: sendBeginTelemetryIfNeeded called');
+
+		if (this._beginTelemetrySent) {
+			// Already sent or in progress
+			return this._beginTelemetryPromise;
+		}
+
+		this._beginTelemetrySent = true;
+		this._beginTelemetryPromise = this._sendRepoInfoTelemetry('begin');
+
+		return this._beginTelemetryPromise.catch((error) => {
+			this._logService.warn(`Failed to send begin repo info telemetry ${error}`);
+		});
+	}
+
+	public async sendEndTelemetry(): Promise<void> {
+		// IANHU: Remove
+		console.log('repoInfoTelemetry: sendEndTelemetry called');
+
+		await this._beginTelemetryPromise;
+
+		return this._sendRepoInfoTelemetry('end').catch((error) => {
+			this._logService.warn(`Failed to send end repo info telemetry ${error}`);
+		});
+	}
+
+	private async _sendRepoInfoTelemetry(location: 'begin' | 'end'): Promise<void> {
+		// IANHU: Remove
+		console.log('repoInfoTelemetry: sendRepoInfoTelemetry called', location);
+
+		if (this._copilotTokenStore.copilotToken?.isInternal !== true) {
+			return;
+		}
+
+		const gitInfo = await this._getRepoInfoTelemetry();
+		if (!gitInfo) {
+			return;
+		}
+
+		const properties = multiplexProperties({
+			...gitInfo,
+			location,
+			telemetryMessageId: this._telemetryMessageId
+		} as RepoInfoInternalTelemetryProperties);
+
+		this._telemetryService.sendInternalMSFTTelemetryEvent('request.repoInfo', properties);
+
+
+		// IANHU: Remove logging later
+		console.log(JSON.stringify({
+			name: 'request.repoInfo',
+			data: {
+				...gitInfo,
+				location,
+				telemetryMessageId: this._telemetryMessageId
+			}
+		}));
+	}
+
+	private async _getRepoInfoTelemetry(): Promise<RepoInfoTelemetryProperties | undefined> {
+		const repoContext = this._gitService.activeRepository.get();
+
+		if (!repoContext || !repoContext.changes) {
+			return;
+		}
+
+		const githubInfo = getGitHubRepoInfoFromContext(repoContext);
+		if (!githubInfo) {
+			return;
+		}
+
+		// Get the upstream commit from the repository
+		const gitAPI = this._gitExtensionService.getExtensionApi();
+		const repository = gitAPI?.getRepository(repoContext.rootUri);
+		const upstreamCommit = repository?.state.HEAD?.upstream?.commit;
+		if (!upstreamCommit) {
+			return;
+		}
+
+		// Before we calculate our async diffs, sign up for file system change events
+		// Any changes during the async operations will invalidate our diff data and we send it
+		// as a failure without a diffs
+		const watcher = this._fileSystemService.createFileSystemWatcher('**/*');
+		let filesChanged = false;
+		const createDisposable = watcher.onDidCreate(() => filesChanged = true);
+		const changeDisposable = watcher.onDidChange(() => filesChanged = true);
+		const deleteDisposable = watcher.onDidDelete(() => filesChanged = true);
+
+		try {
+			const changes = await this._gitService.diffWith(repoContext.rootUri, '@{upstream}');
+			if (!changes || changes.length === 0) {
+				return;
+			}
+
+			// Check if files changed during the git diff operation
+			if (filesChanged) {
+				return {
+					remoteUrl: githubInfo.remoteUrl,
+					headCommitHash: upstreamCommit,
+					diffsJSON: undefined,
+					result: 'filesChanged',
+				};
+			}
+
+			const diffs = (await this._gitDiffService.getChangeDiffs(repoContext.rootUri, changes)).map(diff => {
+				return {
+					uri: diff.uri.toString(),
+					originalUri: diff.originalUri.toString(),
+					renameUri: diff.renameUri?.toString(),
+					status: diff.status,
+					diff: diff.diff,
+				};
+			});
+
+			// Final check if files changed during the diff processing
+			if (filesChanged) {
+				return {
+					remoteUrl: githubInfo.remoteUrl,
+					headCommitHash: upstreamCommit,
+					diffsJSON: undefined,
+					result: 'filesChanged',
+				};
+			}
+
+			const diffsJSON = diffs.length > 0 ? JSON.stringify(diffs) : undefined;
+
+			// Check if the diff is too big and notify that
+			if (wouldMultiplexTelemetryPropertyBeTruncated(diffsJSON)) {
+				return {
+					remoteUrl: githubInfo.remoteUrl,
+					headCommitHash: upstreamCommit,
+					diffsJSON: undefined,
+					result: 'diffTooLarge',
+				};
+			}
+
+			return {
+				remoteUrl: githubInfo.remoteUrl,
+				headCommitHash: upstreamCommit,
+				diffsJSON,
+				result: 'success',
+			};
+		} finally {
+			createDisposable.dispose();
+			changeDisposable.dispose();
+			deleteDisposable.dispose();
+			watcher.dispose();
+		}
+	}
+}

src/platform/telemetry/common/telemetry.ts

@@ -214,4 +214,8 @@ export function multiplexProperties(properties: { [key: string]: string | undefi
 		}
 	}
 	return newProperties;
+}
+
+export function wouldMultiplexTelemetryPropertyBeTruncated(propertyValue: string | undefined): boolean {
+	return (propertyValue?.length ?? 0) > (MAX_PROPERTY_LENGTH * MAX_CONCATENATED_PROPERTIES);
 }

src/util/common/chatResponseStreamImpl.ts

@@ -124,6 +124,8 @@ export class ChatResponseStreamImpl implements FinalizableChatResponseStream {
 	}
 
 	textEdit(target: Uri, editsOrDone: TextEdit | TextEdit[] | true): void {
+		// IANHU: Just debug logging
+		console.log(`repoInfo textedit ${target.fsPath}`);
 		if (Array.isArray(editsOrDone) || editsOrDone instanceof TextEdit) {
 			this._push(new ChatResponseTextEditPart(target, editsOrDone));
 		} else {