tools: simplify virtual tool enablement (#396)

Make the virtual tool setting configure the threshold value, and unify
EXP behind it.

Author: connor4312

package.json

@@ -2669,13 +2669,15 @@
 						],
 						"description": "%github.copilot.config.summarizeAgentConversationHistory.enabled%"
 					},
-					"github.copilot.chat.virtualTools.enabled": {
-						"type": "boolean",
-						"default": false,
+					"github.copilot.chat.virtualTools.threshold": {
+						"type": "number",
+						"minimum": 0,
+						"maximum": 128,
+						"default": 128,
 						"tags": [
 							"experimental"
 						],
-						"description": "%github.copilot.config.virtualTools.enabled%"
+						"markdownDescription": "%github.copilot.config.virtualTools.threshold%"
 					},
 					"github.copilot.chat.azureModels": {
 						"type": "object",
@@ -3683,4 +3685,4 @@
 		"string_decoder": "npm:[email protected]",
 		"node-gyp": "npm:[email protected]"
 	}
-}
+}

package.nls.json

@@ -133,7 +133,7 @@
 	"github.copilot.config.agent.thinkingTool": "Enables the thinking tool that allows Copilot to think deeply about your request before generating a response in agent mode.",
 	"github.copilot.config.setupTests.enabled": "Enables the `/setupTests` intent and prompting in `/tests` generation.",
 	"github.copilot.config.byok.ollamaEndpoint": "The endpoint to use for the Ollama when accessed via bring your own key. Defaults to localhost.",
-	"github.copilot.config.virtualTools.enabled": "Enables the virtual tools feature.",
+	"github.copilot.config.virtualTools.threshold": "This setting defines the tool count over which virtual tools should be used. Virtual tools group similar sets of tools together and they allow the model to activate them on-demand. Certain tool groups will optimistically be pre-activated. We are actively developing this feature and you experience degraded tool calling once the threshold is hit.\n\nMay be set to `0` to disable virtual tools.",
 	"github.copilot.command.fixTestFailure": "Fix Test Failure",
 	"copilot.description": "Ask Copilot",
 	"copilot.edits.description": "Edit files in your workspace",

src/extension/tools/common/virtualTools/toolGrouping.ts

@@ -10,24 +10,19 @@ import { ITelemetryService } from '../../../../platform/telemetry/common/telemet
 import { equals as arraysEqual } from '../../../../util/vs/base/common/arrays';
 import { CancellationToken } from '../../../../util/vs/base/common/cancellation';
 import { Iterable } from '../../../../util/vs/base/common/iterator';
+import { IObservable } from '../../../../util/vs/base/common/observableInternal';
 import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation';
 import { LanguageModelTextPart, LanguageModelToolResult } from '../../../../vscodeTypes';
 import { VIRTUAL_TOOL_NAME_PREFIX, VirtualTool } from './virtualTool';
 import { VirtualToolGrouper } from './virtualToolGrouper';
 import * as Constant from './virtualToolsConstants';
 import { IToolCategorization, IToolGrouping } from './virtualToolTypes';
 
-export function computeToolGroupingMinThreshold(experimentationService: IExperimentationService, configurationService: IConfigurationService): number {
-	let threshold = HARD_TOOL_LIMIT;
-
-	const override = experimentationService.getTreatmentVariable<number>('vscode', 'copilotchat.virtualToolThreshold');
-	if (override) {
-		threshold = override;
-	} else if (configurationService.getExperimentBasedConfig(ConfigKey.VirtualTools, experimentationService)) {
-		threshold = Constant.START_GROUPING_AFTER_TOOL_COUNT;
-	}
-
-	return threshold;
+export function computeToolGroupingMinThreshold(experimentationService: IExperimentationService, configurationService: IConfigurationService): IObservable<number> {
+	return configurationService.getExperimentBasedConfigObservable(ConfigKey.VirtualToolThreshold, experimentationService).map(configured => {
+		const value = configured ?? HARD_TOOL_LIMIT;
+		return value <= 0 ? Infinity : value;
+	});
 }
 
 export class ToolGrouping implements IToolGrouping {
@@ -51,7 +46,7 @@ export class ToolGrouping implements IToolGrouping {
 	}
 
 	public get isEnabled() {
-		return this._tools.length >= computeToolGroupingMinThreshold(this._experimentationService, this._configurationService);
+		return this._tools.length >= computeToolGroupingMinThreshold(this._experimentationService, this._configurationService).get();
 	}
 
 	constructor(

src/extension/tools/common/virtualTools/toolGroupingService.ts

@@ -6,21 +6,22 @@
 import type { LanguageModelToolInformation } from 'vscode';
 import { IConfigurationService } from '../../../../platform/configuration/common/configurationService';
 import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService';
+import { IObservable } from '../../../../util/vs/base/common/observableInternal';
 import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation';
 import { computeToolGroupingMinThreshold, ToolGrouping } from './toolGrouping';
 import { IToolGrouping, IToolGroupingService } from './virtualToolTypes';
 
 export class ToolGroupingService implements IToolGroupingService {
 	declare readonly _serviceBrand: undefined;
 
+	public threshold: IObservable<number>;
+
 	constructor(
 		@IInstantiationService private readonly _instantiationService: IInstantiationService,
-		@IConfigurationService private readonly _configurationService: IConfigurationService,
-		@IExperimentationService private readonly _experimentationService: IExperimentationService
-	) { }
-
-	public get threshold() {
-		return computeToolGroupingMinThreshold(this._experimentationService, this._configurationService);
+		@IConfigurationService configurationService: IConfigurationService,
+		@IExperimentationService experimentationService: IExperimentationService
+	) {
+		this.threshold = computeToolGroupingMinThreshold(experimentationService, configurationService);
 	}
 
 	create(tools: readonly LanguageModelToolInformation[]): IToolGrouping {

src/extension/tools/common/virtualTools/virtualToolTypes.ts

@@ -7,6 +7,7 @@ import type { LanguageModelToolInformation, LanguageModelToolResult } from 'vsco
 import { createServiceIdentifier } from '../../../../util/common/services';
 import { CancellationToken } from '../../../../util/vs/base/common/cancellation';
 import { VirtualTool } from './virtualTool';
+import { IObservable } from '../../../../util/vs/base/common/observableInternal';
 
 export interface IToolGrouping {
 	/**
@@ -60,7 +61,7 @@ export interface IToolGroupingService {
 	/**
 	 * The current tool count threshold for grouping to kick in.
 	 */
-	threshold: number;
+	threshold: IObservable<number>;
 	/**
 	 * Creates a tool grouping for a request, based on its conversation and the
 	 * initial set of tools.

src/extension/tools/vscode-node/tools.ts

@@ -5,11 +5,11 @@
 
 import * as vscode from 'vscode';
 import { Disposable } from '../../../util/vs/base/common/lifecycle';
+import { autorun } from '../../../util/vs/base/common/observableInternal';
 import { getContributedToolName } from '../common/toolNames';
 import { IToolsService } from '../common/toolsService';
-
-import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService';
 import { IToolGroupingCache, IToolGroupingService } from '../common/virtualTools/virtualToolTypes';
+
 import '../node/allTools';
 import './allTools';
 
@@ -18,7 +18,6 @@ export class ToolsContribution extends Disposable {
 		@IToolsService toolsService: IToolsService,
 		@IToolGroupingCache toolGrouping: IToolGroupingCache,
 		@IToolGroupingService toolGroupingService: IToolGroupingService,
-		@IExperimentationService experimentationService: IExperimentationService
 	) {
 		super();
 
@@ -31,8 +30,8 @@ export class ToolsContribution extends Disposable {
 			vscode.window.showInformationMessage('Tool groups have been reset. They will be regenerated on the next agent request.');
 		}));
 
-		experimentationService.initializePromise.then(() => {
-			vscode.commands.executeCommand('setContext', 'chat.toolGroupingThreshold', toolGroupingService.threshold);
-		});
+		this._register(autorun(reader => {
+			vscode.commands.executeCommand('setContext', 'chat.toolGroupingThreshold', toolGroupingService.threshold.read(reader));
+		}));
 	}
 }

src/platform/configuration/common/configurationService.ts

@@ -767,7 +767,7 @@ export namespace ConfigKey {
 	export const NewWorkspaceCreationAgentEnabled = defineSetting<boolean>('chat.newWorkspaceCreation.enabled', true);
 	export const NewWorkspaceUseContext7 = defineSetting<boolean>('chat.newWorkspace.useContext7', false);
 	export const SummarizeAgentConversationHistory = defineExpSetting<boolean>('chat.summarizeAgentConversationHistory.enabled', true);
-	export const VirtualTools = defineExpSetting<boolean>('chat.virtualTools.enabled', false);
+	export const VirtualToolThreshold = defineExpSetting<number>('chat.virtualTools.threshold', HARD_TOOL_LIMIT);
 	export const CurrentEditorAgentContext = defineSetting<boolean>('chat.agent.currentEditorContext.enabled', true);
 	/** BYOK  */
 	export const OllamaEndpoint = defineSetting<string>('chat.byok.ollamaEndpoint', 'http://localhost:11434');

test/base/extHostContext/simulationExtHostToolsService.ts

@@ -14,10 +14,10 @@ import { ToolsContribution } from '../../../src/extension/tools/vscode-node/tool
 import { ToolsService } from '../../../src/extension/tools/vscode-node/toolsService';
 import { packageJson } from '../../../src/platform/env/common/packagejson';
 import { ILogService } from '../../../src/platform/log/common/logService';
-import { NullExperimentationService } from '../../../src/platform/telemetry/common/nullExperimentationService';
 import { raceTimeout } from '../../../src/util/vs/base/common/async';
 import { CancellationError } from '../../../src/util/vs/base/common/errors';
 import { Iterable } from '../../../src/util/vs/base/common/iterator';
+import { observableValue } from '../../../src/util/vs/base/common/observableInternal';
 import { IInstantiationService } from '../../../src/util/vs/platform/instantiation/common/instantiation';
 import { logger } from '../../simulationLogger';
 
@@ -64,7 +64,7 @@ export class SimulationExtHostToolsService extends BaseToolsService implements I
 	}
 
 	private ensureToolsRegistered() {
-		this._lmToolRegistration ??= new ToolsContribution(this, {} as any, { threshold: Infinity } as any, new NullExperimentationService());
+		this._lmToolRegistration ??= new ToolsContribution(this, {} as any, { threshold: observableValue(this, 128) } as any);
 	}
 
 	getCopilotTool(name: string): ICopilotTool<any> | undefined {