Support A/B Compiler Arguments Traits

kuchungmsft · kuchungmsft · commit ce1c8fe6b00b · 2024-11-19T16:23:28.000-08:00
- Depends on cpptools' update to provide ProjectContextResult.
- Added the following new traits
  - intellisense: compiler information disclaimer.
  - intellisenseBegin: to note the beginning of IntelliSense information.
  - compilerArguments: a list of compiler command arguments that could affect Copilot generating completions.
  - directAsks: direct asking Copilot to do something instead of providing an argument.
  - intellisenseEnd: to note the end of IntelliSense information.
- A/B Experimental flags
  - copilotcppTraits: boolean flag to enable cpp traits
  - copilotcppExcludeTraits: string array to exclude individual trait, i.e., compilerArguments.
  - copilotcppMsvcCompilerArgumentFilter: regex string to match compiler arguments for GCC.
  - copilotcppClangCompilerArgumentFilter: regex string to match compiler arguments for Clang.
  - copilotcppGccCompilerArgumentFilter: regex string to match compiler arguments for MSVC.
  - copilotcppCompilerArgumentDirectAskMap: a stringify map string to map arguments to direct ask statements.
diff --git a/Extension/src/LanguageServer/client.ts b/Extension/src/LanguageServer/client.ts
@@ -541,6 +541,19 @@ export interface ChatContextResult {
     targetArchitecture: string;
 }
 
+export interface FileContextResult {
+    compilerArguments: string[];
+}
+
+export interface ProjectContextResult {
+    language: string;
+    standardVersion: string;
+    compiler: string;
+    targetPlatform: string;
+    targetArchitecture: string;
+    fileContext: FileContextResult;
+}
+
 // Requests
 const PreInitializationRequest: RequestType<void, string, void> = new RequestType<void, string, void>('cpptools/preinitialize');
 const InitializationRequest: RequestType<CppInitializationParams, void, void> = new RequestType<CppInitializationParams, void, void>('cpptools/initialize');
@@ -561,6 +574,7 @@ const GenerateDoxygenCommentRequest: RequestType<GenerateDoxygenCommentParams, G
 const ChangeCppPropertiesRequest: RequestType<CppPropertiesParams, void, void> = new RequestType<CppPropertiesParams, void, void>('cpptools/didChangeCppProperties');
 const IncludesRequest: RequestType<GetIncludesParams, GetIncludesResult, void> = new RequestType<GetIncludesParams, GetIncludesResult, void>('cpptools/getIncludes');
 const CppContextRequest: RequestType<void, ChatContextResult, void> = new RequestType<void, ChatContextResult, void>('cpptools/getChatContext');
+const ProjectContextRequest: RequestType<void, ProjectContextResult, void> = new RequestType<void, ProjectContextResult, void>('cpptools/getProjectContext');
 
 // Notifications to the server
 const DidOpenNotification: NotificationType<DidOpenTextDocumentParams> = new NotificationType<DidOpenTextDocumentParams>('textDocument/didOpen');
@@ -792,6 +806,7 @@ export interface Client {
     addTrustedCompiler(path: string): Promise<void>;
     getIncludes(maxDepth: number, token: vscode.CancellationToken): Promise<GetIncludesResult>;
     getChatContext(token: vscode.CancellationToken): Promise<ChatContextResult>;
+    getProjectContext(token: vscode.CancellationToken): Promise<ProjectContextResult>;
 }
 
 export function createClient(workspaceFolder?: vscode.WorkspaceFolder): Client {
@@ -2220,6 +2235,12 @@ export class DefaultClient implements Client {
             () => this.languageClient.sendRequest(CppContextRequest, null, token), token);
     }
 
+    public async getProjectContext(token: vscode.CancellationToken): Promise<ProjectContextResult> {
+        await withCancellation(this.ready, token);
+        return DefaultClient.withLspCancellationHandling(
+            () => this.languageClient.sendRequest(ProjectContextRequest, null, token), token);
+    }
+
     /**
      * a Promise that can be awaited to know when it's ok to proceed.
      *
@@ -4123,4 +4144,5 @@ class NullClient implements Client {
     addTrustedCompiler(path: string): Promise<void> { return Promise.resolve(); }
     getIncludes(maxDepth: number, token: vscode.CancellationToken): Promise<GetIncludesResult> { return Promise.resolve({} as GetIncludesResult); }
     getChatContext(token: vscode.CancellationToken): Promise<ChatContextResult> { return Promise.resolve({} as ChatContextResult); }
+    getProjectContext(token: vscode.CancellationToken): Promise<ProjectContextResult> { return Promise.resolve({} as ProjectContextResult); }
 }
diff --git a/Extension/src/LanguageServer/copilotProviders.ts b/Extension/src/LanguageServer/copilotProviders.ts
@@ -6,8 +6,9 @@
 
 import * as vscode from 'vscode';
 import * as util from '../common';
-import { ChatContextResult, GetIncludesResult } from './client';
+import { GetIncludesResult } from './client';
 import { getActiveClient } from './extension';
+import { getProjectContext } from './lmTool';
 
 export interface CopilotTrait {
     name: string;
@@ -38,19 +39,51 @@ export async function registerRelatedFilesProvider(): Promise<void> {
 
                         const getIncludesHandler = async () => (await getIncludesWithCancellation(1, token))?.includedFiles.map(file => vscode.Uri.file(file)) ?? [];
                         const getTraitsHandler = async () => {
-                            const chatContext: ChatContextResult | undefined = await (getActiveClient().getChatContext(token) ?? undefined);
+                            const cppContext = await getProjectContext(context, token);
 
-                            if (!chatContext) {
+                            if (!cppContext) {
                                 return undefined;
                             }
 
                             let traits: CopilotTrait[] = [
-                                { name: "language", value: chatContext.language, includeInPrompt: true, promptTextOverride: `The language is ${chatContext.language}.` },
-                                { name: "compiler", value: chatContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${chatContext.compiler}.` },
-                                { name: "standardVersion", value: chatContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${chatContext.standardVersion} language standard.` },
-                                { name: "targetPlatform", value: chatContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetPlatform}.` },
-                                { name: "targetArchitecture", value: chatContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetArchitecture}.` }
+                                { name: "intellisense", value: 'intellisense', includeInPrompt: true, promptTextOverride: `IntelliSense is currently configured with the following compiler information. It reflects the active configuration, and the project may have more configurations targeting different platforms.` },
+                                { name: "intellisenseBegin", value: 'Begin', includeInPrompt: true, promptTextOverride: `Beginning of IntelliSense information.` }
                             ];
+                            if (cppContext.language) {
+                                traits.push({ name: "language", value: cppContext.language, includeInPrompt: true, promptTextOverride: `The language is ${cppContext.language}.` });
+                            }
+                            if (cppContext.compiler) {
+                                traits.push({ name: "compiler", value: cppContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${cppContext.compiler}.` });
+                            }
+                            if (cppContext.standardVersion) {
+                                traits.push({ name: "standardVersion", value: cppContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${cppContext.standardVersion} language standard.` });
+                            }
+                            if (cppContext.targetPlatform) {
+                                traits.push({ name: "targetPlatform", value: cppContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${cppContext.targetPlatform}.` });
+                            }
+                            if (cppContext.targetArchitecture) {
+                                traits.push({ name: "targetArchitecture", value: cppContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${cppContext.targetArchitecture}.` });
+                            }
+                            let directAsks: string = '';
+                            if (cppContext.compilerArguments.length > 0) {
+                                // Example: JSON.stringify({'-fno-rtti': "Do not generate code using RTTI keywords."})
+                                const directAskMap: { [key: string]: string } = JSON.parse(context.flags.copilotcppCompilerArgumentDirectAskMap as string ?? '{}');
+                                const updatedArguments = cppContext.compilerArguments.filter(arg => {
+                                    if (directAskMap[arg]) {
+                                        directAsks += `${directAskMap[arg]} `;
+                                        return false;
+                                    }
+                                    return true;
+                                });
+
+                                const compilerArgumentsValue = updatedArguments.join(", ");
+                                traits.push({ name: "compilerArguments", value: compilerArgumentsValue, includeInPrompt: true, promptTextOverride: `The compiler arguments include: ${compilerArgumentsValue}.` });
+                            }
+                            if (directAsks) {
+                                traits.push({ name: "directAsks", value: directAsks, includeInPrompt: true, promptTextOverride: directAsks });
+                            }
+
+                            traits.push({ name: "intellisenseEnd", value: 'End', includeInPrompt: true, promptTextOverride: `End of IntelliSense information.` });
 
                             const excludeTraits = context.flags.copilotcppExcludeTraits as string[] ?? [];
                             traits = traits.filter(trait => !excludeTraits.includes(trait.name));
diff --git a/Extension/src/LanguageServer/lmTool.ts b/Extension/src/LanguageServer/lmTool.ts
@@ -9,19 +9,23 @@ import { localize } from 'vscode-nls';
 import * as util from '../common';
 import * as logger from '../logger';
 import * as telemetry from '../telemetry';
-import { ChatContextResult } from './client';
+import { ChatContextResult, ProjectContextResult } from './client';
 import { getClients } from './extension';
+import { checkTime } from './utils';
 
+const MSVC: string = 'MSVC';
+const Clang: string = 'Clang';
+const GCC: string = 'GCC';
 const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: string } } = {
     language: {
         'c': 'C',
         'cpp': 'C++',
         'cuda-cpp': 'CUDA C++'
     },
     compiler: {
-        'msvc': 'MSVC',
-        'clang': 'Clang',
-        'gcc': 'GCC'
+        'msvc': MSVC,
+        'clang': Clang,
+        'gcc': GCC
     },
     standardVersion: {
         'c++98': 'C++98',
@@ -44,6 +48,112 @@ const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: str
     }
 };
 
+function formatChatContext(context: ChatContextResult | ProjectContextResult): void {
+    type KnownKeys = 'language' | 'standardVersion' | 'compiler' | 'targetPlatform';
+    for (const key in knownValues) {
+        const knownKey = key as KnownKeys;
+        if (knownValues[knownKey] && context[knownKey]) {
+            // Clear the value if it's not in the known values.
+            context[knownKey] = knownValues[knownKey][context[knownKey]] || "";
+        }
+    }
+}
+
+export interface ProjectContext {
+    language: string;
+    standardVersion: string;
+    compiler: string;
+    targetPlatform: string;
+    targetArchitecture: string;
+    compilerArguments: string[];
+}
+
+// Set these values for local testing purpose without involving control tower.
+const defaultCompilerArgumentFilters: { [id: string]: RegExp | undefined } = {
+    MSVC: undefined, // Example: /^(\/std:.*|\/EHs-c-|\/GR-|\/await.*)$/,
+    Clang: undefined,
+    GCC: undefined // Example: /^(-std=.*|-fno-rtti|-fno-exceptions)$/
+};
+
+function filterComplierArguments(compiler: string, compilerArguments: string[], context: { flags: Record<string, unknown> }): string[] {
+    const defaultFilter: RegExp | undefined = defaultCompilerArgumentFilters[compiler] ?? undefined;
+    let additionalFilter: RegExp | undefined;
+    switch (compiler) {
+        case MSVC:
+            additionalFilter = context.flags.copilotcppMsvcCompilerArgumentFilter ? new RegExp(context.flags.copilotcppMsvcCompilerArgumentFilter as string) : undefined;
+            break;
+        case Clang:
+            additionalFilter = context.flags.copilotcppClangCompilerArgumentFilter ? new RegExp(context.flags.copilotcppClangCompilerArgumentFilter as string) : undefined;
+            break;
+        case GCC:
+            additionalFilter = context.flags.copilotcppGccCompilerArgumentFilter ? new RegExp(context.flags.copilotcppGccCompilerArgumentFilter as string) : undefined;
+            break;
+    }
+
+    return compilerArguments.filter(arg => defaultFilter?.test(arg) || additionalFilter?.test(arg));
+}
+
+export async function getProjectContext(context: { flags: Record<string, unknown> }, token: vscode.CancellationToken): Promise<ProjectContext | undefined> {
+    const telemetryProperties: Record<string, string> = {};
+    try {
+        const projectContext = await checkTime<ProjectContextResult | undefined>(async () => await getClients()?.ActiveClient?.getProjectContext(token) ?? undefined);
+        telemetryProperties["time"] = projectContext.time.toString();
+        if (!projectContext.result) {
+            return undefined;
+        }
+
+        formatChatContext(projectContext.result);
+
+        const result: ProjectContext = {
+            language: projectContext.result.language,
+            standardVersion: projectContext.result.standardVersion,
+            compiler: projectContext.result.compiler,
+            targetPlatform: projectContext.result.targetPlatform,
+            targetArchitecture: projectContext.result.targetArchitecture,
+            compilerArguments: []
+        };
+
+        if (projectContext.result.language) {
+            telemetryProperties["language"] = projectContext.result.language;
+        }
+        if (projectContext.result.compiler) {
+            telemetryProperties["compiler"] = projectContext.result.compiler;
+        }
+        if (projectContext.result.standardVersion) {
+            telemetryProperties["standardVersion"] = projectContext.result.standardVersion;
+        }
+        if (projectContext.result.targetPlatform) {
+            telemetryProperties["targetPlatform"] = projectContext.result.targetPlatform;
+        }
+        if (projectContext.result.targetArchitecture) {
+            telemetryProperties["targetArchitecture"] = projectContext.result.targetArchitecture;
+        }
+        telemetryProperties["compilerArgumentCount"] = projectContext.result.fileContext.compilerArguments.length.toString();
+        // Telemtry to learn about the argument distribution. The filtered arguments are expected to be non-PII.
+        if (projectContext.result.fileContext.compilerArguments.length) {
+            const filteredCompilerArguments = filterComplierArguments(projectContext.result.compiler, projectContext.result.fileContext.compilerArguments, context);
+            if (filteredCompilerArguments.length > 0) {
+                telemetryProperties["filteredCompilerArguments"] = filteredCompilerArguments.join(', ');
+                result.compilerArguments = filteredCompilerArguments;
+            }
+        }
+
+        return result;
+    }
+    catch {
+        try {
+            logger.getOutputChannelLogger().appendLine(localize("copilot.cppcontext.error", "Error while retrieving the project context."));
+        }
+        catch {
+            // Intentionally swallow any exception.
+        }
+        telemetryProperties["error"] = "true";
+        return undefined;
+    } finally {
+        telemetry.logLanguageModelToolEvent('Completions/tool', telemetryProperties);
+    }
+}
+
 export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTool<void> {
     public async invoke(options: vscode.LanguageModelToolInvocationOptions<void>, token: vscode.CancellationToken): Promise<vscode.LanguageModelToolResult> {
         return new vscode.LanguageModelToolResult([
@@ -63,13 +173,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo
                 return 'No configuration information is available for the active document.';
             }
 
-            for (const key in knownValues) {
-                const knownKey = key as keyof ChatContextResult;
-                if (knownValues[knownKey] && chatContext[knownKey]) {
-                    // Clear the value if it's not in the known values.
-                    chatContext[knownKey] = knownValues[knownKey][chatContext[knownKey]] || "";
-                }
-            }
+            formatChatContext(chatContext);
 
             let contextString = "";
             if (chatContext.language) {
@@ -100,7 +204,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo
             telemetryProperties["error"] = "true";
             return "";
         } finally {
-            telemetry.logLanguageModelToolEvent('cpp', telemetryProperties);
+            telemetry.logLanguageModelToolEvent('Chat/Tool/cpp', telemetryProperties);
         }
     }
 
diff --git a/Extension/src/LanguageServer/utils.ts b/Extension/src/LanguageServer/utils.ts
@@ -112,3 +112,9 @@ export async function withCancellation<T>(promise: Promise<T>, token: vscode.Can
         });
     });
 }
+
+export async function checkTime<T>(fn: () => Promise<T>): Promise<{ result: T; time: number }> {
+    const start = Date.now();
+    const result = await fn();
+    return { result, time: Date.now() - start };
+}
diff --git a/Extension/src/telemetry.ts b/Extension/src/telemetry.ts
@@ -126,7 +126,7 @@ export function logLanguageServerEvent(eventName: string, properties?: Record<st
 export function logLanguageModelToolEvent(eventName: string, properties?: Record<string, string>, metrics?: Record<string, number>): void {
     const sendTelemetry = () => {
         if (experimentationTelemetry) {
-            const eventNamePrefix: string = "C_Cpp/Copilot/Chat/Tool/";
+            const eventNamePrefix: string = "C_Cpp/Copilot/";
             experimentationTelemetry.sendTelemetryEvent(eventNamePrefix + eventName, properties, metrics);
         }
     };
diff --git a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts
diff --git a/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts

Original file line number	Diff line number	Diff line change
`@@ -126,7 +126,7 @@ export function logLanguageServerEvent(eventName: string, properties?: Record<st`
`126`	`126`	`export function logLanguageModelToolEvent(eventName: string, properties?: Record<string, string>, metrics?: Record<string, number>): void {`
`127`	`127`	`const sendTelemetry = () => {`
`128`	`128`	`if (experimentationTelemetry) {`
`129`		`- const eventNamePrefix: string = "C_Cpp/Copilot/Chat/Tool/";`
	`129`	`+ const eventNamePrefix: string = "C_Cpp/Copilot/";`
`130`	`130`	`experimentationTelemetry.sendTelemetryEvent(eventNamePrefix + eventName, properties, metrics);`
`131`	`131`	`}`
`132`	`132`	`};`