From 8f4b74825e274d13c5982625da4cd9944b750b7e Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Wed, 24 Sep 2025 11:44:47 -0700 Subject: [PATCH 01/10] adding starting changes for tool selection from default toolset --- .../extension/vscode-node/services.ts | 2 +- .../node/defaultIntentRequestHandler.ts | 2 +- .../common/virtualTools/virtualToolGrouper.ts | 140 +++++++++++++++++- .../common/configurationService.ts | 1 + 4 files changed, 138 insertions(+), 7 deletions(-) diff --git a/src/extension/extension/vscode-node/services.ts b/src/extension/extension/vscode-node/services.ts index 04fab1d78..e7afa20a5 100644 --- a/src/extension/extension/vscode-node/services.ts +++ b/src/extension/extension/vscode-node/services.ts @@ -24,6 +24,7 @@ import { IEndpointProvider } from '../../../platform/endpoint/common/endpointPro import { CAPIClientImpl } from '../../../platform/endpoint/node/capiClientImpl'; import { DomainService } from '../../../platform/endpoint/node/domainServiceImpl'; import { INativeEnvService, isScenarioAutomation } from '../../../platform/env/common/envService'; +import { NativeEnvServiceImpl } from '../../../platform/env/vscode-node/nativeEnvServiceImpl'; import { IGitCommitMessageService } from '../../../platform/git/common/gitCommitMessageService'; import { IGitDiffService } from '../../../platform/git/common/gitDiffService'; import { IGithubRepositoryService } from '../../../platform/github/common/githubService'; @@ -102,7 +103,6 @@ import { LanguageContextServiceImpl } from '../../typescriptContext/vscode-node/ import { IWorkspaceListenerService } from '../../workspaceRecorder/common/workspaceListenerService'; import { WorkspacListenerService } from '../../workspaceRecorder/vscode-node/workspaceListenerService'; import { registerServices as registerCommonServices } from '../vscode/services'; -import { NativeEnvServiceImpl } from '../../../platform/env/vscode-node/nativeEnvServiceImpl'; // ########################################################################################### // ### ### diff --git a/src/extension/prompt/node/defaultIntentRequestHandler.ts b/src/extension/prompt/node/defaultIntentRequestHandler.ts index 9a4e481d5..272a59a27 100644 --- a/src/extension/prompt/node/defaultIntentRequestHandler.ts +++ b/src/extension/prompt/node/defaultIntentRequestHandler.ts @@ -42,6 +42,7 @@ import { SummarizedConversationHistoryMetadata } from '../../prompts/node/agent/ import { normalizeToolSchema } from '../../tools/common/toolSchemaNormalizer'; import { ToolCallCancelledError } from '../../tools/common/toolsService'; import { IToolGrouping, IToolGroupingService } from '../../tools/common/virtualTools/virtualToolTypes'; +import { ChatVariablesCollection } from '../common/chatVariablesCollection'; import { Conversation, getUniqueReferences, GlobalContextMessageMetadata, IResultMetadata, RenderedUserMessageMetadata, RequestDebugInformation, ResponseStreamParticipant, Turn, TurnStatus } from '../common/conversation'; import { IBuildPromptContext, IToolCallRound } from '../common/intents'; import { ChatTelemetry, ChatTelemetryBuilder } from './chatParticipantTelemetry'; @@ -49,7 +50,6 @@ import { IntentInvocationMetadata } from './conversation'; import { IDocumentContext } from './documentContext'; import { IBuildPromptResult, IIntent, IIntentInvocation, IResponseProcessor } from './intents'; import { ConversationalBaseTelemetryData, createTelemetryWithId, sendModelMessageTelemetry } from './telemetry'; -import { ChatVariablesCollection } from '../common/chatVariablesCollection'; export interface IDefaultIntentRequestHandlerOptions { maxToolCallIterations: number; diff --git a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts index d1bce5008..d95f61615 100644 --- a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts +++ b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts @@ -28,6 +28,73 @@ const CATEGORIZATION_ENDPOINT = CHAT_MODEL.GPT4OMINI; const SUMMARY_PREFIX = 'Call this tool when you need access to a new category of tools. The category of tools is described as follows:\n\n'; const SUMMARY_SUFFIX = '\n\nBe sure to call this tool if you need a capability related to the above.'; +// categorize all tools except for the 11 default tools +// 11 default tools = semantic_search, grep_search, read_file, create_file, apply_patch, replace_string_in_file, +// insert_edit_into_file, run_in_terminal, list_dir, think, get_terminal_output +const BUILT_IN_TOOL_GROUPS = { + 'Jupyter Notebook Tools': { + // "Call this tool when..." + summary: 'Tools for working with Jupyter notebooks - creating, editing, running cells, and managing notebook operations.', + tools: [ + 'create_new_jupyter_notebook', + 'edit_notebook_file', + 'run_notebook_cell', + 'copilot_getNotebookSummary', + 'read_notebook_cell_output', + 'configure_notebook', + 'notebook_install_packages', + 'notebook_list_packages', + 'configure_python_environment', + 'get_python_environment_details', + 'get_python_executable_details' + ] + }, + 'Web Interaction': { + summary: 'Tools for interacting with web content, browsing websites, and accessing external resources.', + tools: [ + 'fetch_webpage', + 'open_simple_browser', + 'github_repo' + ] + }, + 'VS Code Interaction': { + summary: 'Tools for interacting with VS Code workspace and accessing VS Code features.', + tools: [ + 'search_workspace_symbols', + 'list_code_usages', + 'get_errors', + 'get_vscode_api', + // 'get_changed_files', + 'create_new_workspace', + 'install_extension', + 'get_project_setup_info', + 'create_and_run_task', + 'run_task', + 'get_task_output', + 'run_vscode_command', + // 'file_search', + // 'create_directory', + // 'multi_replace_string_in_file', + // 'terminal_selection', + // 'terminal_last_command', + 'install_python_packages', + // 'manage_todo_list', + 'get_search_view_results', + // 'edit_files', + 'vscode_searchExtensions_internal' + ] + }, + 'Testing': { + summary: 'Tools for running tests, analyzing test failures, and managing test workflows.', + tools: [ + 'run_tests', + 'test_failure', + 'test_search', + 'runTests' + ] + } +} as const; + export class VirtualToolGrouper implements IToolCategorization { private readonly toolEmbeddingsComputer: ToolEmbeddingsComputer; @@ -76,12 +143,9 @@ export class VirtualToolGrouper implements IToolCategorization { const predictedToolsSw = new StopWatch(); const predictedToolsPromise = virtualToolEmbeddingRankingEnabled && this._getPredictedTools(query, tools, token).then(tools => ({ tools, durationMs: predictedToolsSw.elapsed() })); + // Now all toolsets (including built-in) go through _generateGroupsFromToolset const grouped = await Promise.all(Object.entries(byToolset).map(([key, tools]) => { - if (key === BUILT_IN_GROUP) { - return tools; - } else { - return this._generateGroupsFromToolset(key, tools, previousCategorizations.get(key), token); - } + return this._generateGroupsFromToolset(key, tools, previousCategorizations.get(key), token); })); this._cache.flush(); @@ -226,6 +290,16 @@ export class VirtualToolGrouper implements IToolCategorization { /** Top-level request to categorize a group of tools from a single source. */ private async _generateGroupsFromToolset(key: string, tools: LanguageModelToolInformation[], previous: ISummarizedToolCategory[] | undefined, token: CancellationToken): Promise<(VirtualTool | LanguageModelToolInformation)[]> { + // Handle built-in tools with predefined groups only if experimental setting is enabled + if (key === BUILT_IN_GROUP) { + const defaultToolGroupingEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.DefaultToolGrouping, this._expService); + if (defaultToolGroupingEnabled) { + return this._createBuiltInToolGroups(tools); + } else { + return tools; + } + } + if (tools.length <= Constant.MIN_TOOLSET_SIZE_TO_GROUP) { return tools; } @@ -301,6 +375,62 @@ export class VirtualToolGrouper implements IToolCategorization { return virtualTools.concat(uncategorized); } + /** Creates predefined groups for built-in tools */ + private _createBuiltInToolGroups(tools: LanguageModelToolInformation[]): (VirtualTool | LanguageModelToolInformation)[] { + // If there are too few tools, don't group them + if (tools.length <= Constant.MIN_TOOLSET_SIZE_TO_GROUP) { + return tools; + } + + // Create a map of tool names to tool objects for easy lookup + const toolMap = new Map(tools.map(tool => [tool.name, tool])); + + const virtualTools: VirtualTool[] = []; + const usedTools = new Set(); + + // Create virtual tools for each predefined group + for (const [groupName, groupDef] of Object.entries(BUILT_IN_TOOL_GROUPS)) { + const groupTools = groupDef.tools + .map(toolName => toolMap.get(toolName)) + .filter((tool): tool is LanguageModelToolInformation => tool !== undefined); + + if (groupTools.length > 0) { + // Mark these tools as used + groupTools.forEach(tool => usedTools.add(tool.name)); + + // Create the virtual tool group + const virtualTool = new VirtualTool( + VIRTUAL_TOOL_NAME_PREFIX + groupName.toLowerCase().replace(/\s+/g, '_'), + SUMMARY_PREFIX + groupDef.summary + SUMMARY_SUFFIX, + 0, + { + toolsetKey: BUILT_IN_GROUP, + groups: [], + possiblePrefix: 'builtin_' + }, + groupTools + ); + virtualTools.push(virtualTool); + } + } + + // Add any remaining uncategorized tools individually + const uncategorizedTools = tools.filter(tool => !usedTools.has(tool.name)); + + // Send telemetry for built-in tool grouping + this._telemetryService.sendMSFTTelemetryEvent('virtualTools.generate', { + groupKey: BUILT_IN_GROUP, + }, { + uncategorized: uncategorizedTools.length, + toolsBefore: tools.length, + toolsAfter: virtualTools.length, + retries: 0, // No retries for predefined groups + durationMs: 0, // Instant for predefined groups + }); + + return [...virtualTools, ...uncategorizedTools]; + } + private async _getPredictedTools(query: string, tools: LanguageModelToolInformation[], token: CancellationToken): Promise { // compute the embeddings for the query const queryEmbedding = await this.embeddingsComputer.computeEmbeddings(EMBEDDING_TYPE_FOR_TOOL_GROUPING, [query], {}, new TelemetryCorrelationId('VirtualToolGrouper::_getPredictedTools'), token); diff --git a/src/platform/configuration/common/configurationService.ts b/src/platform/configuration/common/configurationService.ts index 7bc0cc87a..2cc317d23 100644 --- a/src/platform/configuration/common/configurationService.ts +++ b/src/platform/configuration/common/configurationService.ts @@ -734,6 +734,7 @@ export namespace ConfigKey { export const PromptFileContext = defineExpSetting('chat.advanced.promptFileContextProvider.enabled', true); export const MultiReplaceString = defineExpSetting('chat.advanced.multiReplaceString.enabled', false, INTERNAL); + export const DefaultToolGrouping = defineExpSetting('chat.advanced.tools.defaultToolsGrouped', false, INTERNAL); export const VirtualToolEmbeddingRanking = defineExpSetting('chat.advanced.virtualTools.embeddingRanking', false, INTERNAL); export const MultiReplaceStringGrok = defineExpSetting('chat.advanced.multiReplaceStringGrok.enabled', false, INTERNAL); export const Gpt5ApplyPatchExclusively = defineExpSetting('chat.advanced.gpt5ApplyPatchExclusively.enabled', false, INTERNAL); From d878d19033cf7b8a269f1edef7bd8ce022388bb1 Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Wed, 24 Sep 2025 20:57:16 -0700 Subject: [PATCH 02/10] update code to have all current default tools in groups (or ungrouped if default) and set exp to true --- .../common/virtualTools/virtualToolGrouper.ts | 38 ++++++++++--------- .../common/configurationService.ts | 4 +- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts index d95f61615..76019792c 100644 --- a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts +++ b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts @@ -29,12 +29,11 @@ const SUMMARY_PREFIX = 'Call this tool when you need access to a new category of const SUMMARY_SUFFIX = '\n\nBe sure to call this tool if you need a capability related to the above.'; // categorize all tools except for the 11 default tools -// 11 default tools = semantic_search, grep_search, read_file, create_file, apply_patch, replace_string_in_file, -// insert_edit_into_file, run_in_terminal, list_dir, think, get_terminal_output +// 12 default tools = semantic_search, grep_search, read_file, create_file, apply_patch, replace_string_in_file, +// insert_edit_into_file, run_in_terminal, list_dir, think, get_terminal_output, manage_todo_list const BUILT_IN_TOOL_GROUPS = { 'Jupyter Notebook Tools': { - // "Call this tool when..." - summary: 'Tools for working with Jupyter notebooks - creating, editing, running cells, and managing notebook operations.', + summary: 'Call tools from this group when you need to work with Jupyter notebooks - creating, editing, running cells, and managing notebook operations.', tools: [ 'create_new_jupyter_notebook', 'edit_notebook_file', @@ -50,7 +49,7 @@ const BUILT_IN_TOOL_GROUPS = { ] }, 'Web Interaction': { - summary: 'Tools for interacting with web content, browsing websites, and accessing external resources.', + summary: 'Call tools from this group when you need to interact with web content, browse websites, or access external resources.', tools: [ 'fetch_webpage', 'open_simple_browser', @@ -58,13 +57,13 @@ const BUILT_IN_TOOL_GROUPS = { ] }, 'VS Code Interaction': { - summary: 'Tools for interacting with VS Code workspace and accessing VS Code features.', + summary: 'Call tools from this group when you need to interact with the VS Code workspace and access VS Code features.', tools: [ 'search_workspace_symbols', 'list_code_usages', 'get_errors', 'get_vscode_api', - // 'get_changed_files', + 'get_changed_files', 'create_new_workspace', 'install_extension', 'get_project_setup_info', @@ -72,26 +71,31 @@ const BUILT_IN_TOOL_GROUPS = { 'run_task', 'get_task_output', 'run_vscode_command', - // 'file_search', - // 'create_directory', - // 'multi_replace_string_in_file', - // 'terminal_selection', - // 'terminal_last_command', + 'multi_replace_string_in_file', 'install_python_packages', - // 'manage_todo_list', 'get_search_view_results', - // 'edit_files', - 'vscode_searchExtensions_internal' + 'vscode_searchExtensions_internal', + 'read_project_structure' ] }, 'Testing': { - summary: 'Tools for running tests, analyzing test failures, and managing test workflows.', + summary: 'Call tools from this group when you need to run tests, analyze test failures, and manage test workflows.', tools: [ 'run_tests', 'test_failure', 'test_search', 'runTests' ] + }, + 'Redundant but Specific': { + summary: 'These tools have overlapping functionalities but are highly specialized for certain tasks. \nTools: file_search, get_terminal_selection, get_terminal_last_command, create_directory, get_doc_info', + tools: [ + 'file_search', + 'get_terminal_selection', + 'get_terminal_last_command', + 'create_directory', + 'get_doc_info' + ] } } as const; @@ -292,7 +296,7 @@ export class VirtualToolGrouper implements IToolCategorization { private async _generateGroupsFromToolset(key: string, tools: LanguageModelToolInformation[], previous: ISummarizedToolCategory[] | undefined, token: CancellationToken): Promise<(VirtualTool | LanguageModelToolInformation)[]> { // Handle built-in tools with predefined groups only if experimental setting is enabled if (key === BUILT_IN_GROUP) { - const defaultToolGroupingEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.DefaultToolGrouping, this._expService); + const defaultToolGroupingEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.DefaultToolsGrouped, this._expService); if (defaultToolGroupingEnabled) { return this._createBuiltInToolGroups(tools); } else { diff --git a/src/platform/configuration/common/configurationService.ts b/src/platform/configuration/common/configurationService.ts index 2cc317d23..c4729df7d 100644 --- a/src/platform/configuration/common/configurationService.ts +++ b/src/platform/configuration/common/configurationService.ts @@ -522,7 +522,7 @@ export function defineExpSetting(key: strin } // Max CAPI tool count limit -export const HARD_TOOL_LIMIT = 128; +export const HARD_TOOL_LIMIT = 24; // WARNING // These values are used in the request and are case sensitive. Do not change them unless advised by CAPI. @@ -734,7 +734,7 @@ export namespace ConfigKey { export const PromptFileContext = defineExpSetting('chat.advanced.promptFileContextProvider.enabled', true); export const MultiReplaceString = defineExpSetting('chat.advanced.multiReplaceString.enabled', false, INTERNAL); - export const DefaultToolGrouping = defineExpSetting('chat.advanced.tools.defaultToolsGrouped', false, INTERNAL); + export const DefaultToolsGrouped = defineExpSetting('chat.advanced.tools.defaultToolsGrouped', true, INTERNAL); export const VirtualToolEmbeddingRanking = defineExpSetting('chat.advanced.virtualTools.embeddingRanking', false, INTERNAL); export const MultiReplaceStringGrok = defineExpSetting('chat.advanced.multiReplaceStringGrok.enabled', false, INTERNAL); export const Gpt5ApplyPatchExclusively = defineExpSetting('chat.advanced.gpt5ApplyPatchExclusively.enabled', false, INTERNAL); From ab16c1fe0532423dc6728115eb6662da6fc04832 Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Thu, 25 Sep 2025 08:12:52 -0700 Subject: [PATCH 03/10] update max tools --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index ea310e888..80facda87 100644 --- a/package.json +++ b/package.json @@ -2895,8 +2895,8 @@ "github.copilot.chat.virtualTools.threshold": { "type": "number", "minimum": 0, - "maximum": 128, - "default": 128, + "maximum": 24, + "default": 24, "tags": [ "experimental" ], From 422b3a5269d74de52ee7a703cea912d02c528cf6 Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Thu, 25 Sep 2025 11:32:46 -0700 Subject: [PATCH 04/10] small changes --- src/extension/tools/common/virtualTools/virtualToolGrouper.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts index 76019792c..70a7306d8 100644 --- a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts +++ b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts @@ -379,7 +379,7 @@ export class VirtualToolGrouper implements IToolCategorization { return virtualTools.concat(uncategorized); } - /** Creates predefined groups for built-in tools */ + /** Creates groups for built-in tools based on the pre-defined enum above*/ private _createBuiltInToolGroups(tools: LanguageModelToolInformation[]): (VirtualTool | LanguageModelToolInformation)[] { // If there are too few tools, don't group them if (tools.length <= Constant.MIN_TOOLSET_SIZE_TO_GROUP) { @@ -399,7 +399,7 @@ export class VirtualToolGrouper implements IToolCategorization { .filter((tool): tool is LanguageModelToolInformation => tool !== undefined); if (groupTools.length > 0) { - // Mark these tools as used + // Mark each tool that has already been added to a group groupTools.forEach(tool => usedTools.add(tool.name)); // Create the virtual tool group From 9c600ac6e813c53e54f3f2b09752dd648e78f06e Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Thu, 25 Sep 2025 11:52:51 -0700 Subject: [PATCH 05/10] separate logic for built in tool grouping to new file --- .../virtualTools/builtInToolGroupHandler.ts | 160 ++++++++++++++++++ .../common/virtualTools/virtualToolGrouper.ts | 145 ++-------------- 2 files changed, 175 insertions(+), 130 deletions(-) create mode 100644 src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts diff --git a/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts b/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts new file mode 100644 index 000000000..e3b262862 --- /dev/null +++ b/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts @@ -0,0 +1,160 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { LanguageModelToolInformation } from 'vscode'; +import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry'; +import { VIRTUAL_TOOL_NAME_PREFIX, VirtualTool } from './virtualTool'; +import * as Constant from './virtualToolsConstants'; + +const BUILT_IN_GROUP = 'builtin'; +const SUMMARY_PREFIX = 'Call this tool when you need access to a new category of tools. The category of tools is described as follows:\n\n'; +const SUMMARY_SUFFIX = '\n\nBe sure to call this tool if you need a capability related to the above.'; + +// categorize all tools except for the 11 default tools +// 12 default tools = semantic_search, grep_search, read_file, create_file, apply_patch, replace_string_in_file, +// insert_edit_into_file, run_in_terminal, list_dir, think, get_terminal_output, manage_todo_list +const BUILT_IN_TOOL_GROUPS = { + 'Jupyter Notebook Tools': { + summary: 'Call tools from this group when you need to work with Jupyter notebooks - creating, editing, running cells, and managing notebook operations.', + tools: [ + 'create_new_jupyter_notebook', + 'edit_notebook_file', + 'run_notebook_cell', + 'copilot_getNotebookSummary', + 'read_notebook_cell_output', + 'configure_notebook', + 'notebook_install_packages', + 'notebook_list_packages', + 'configure_python_environment', + 'get_python_environment_details', + 'get_python_executable_details' + ] + }, + 'Web Interaction': { + summary: 'Call tools from this group when you need to interact with web content, browse websites, or access external resources.', + tools: [ + 'fetch_webpage', + 'open_simple_browser', + 'github_repo' + ] + }, + 'VS Code Interaction': { + summary: 'Call tools from this group when you need to interact with the VS Code workspace and access VS Code features.', + tools: [ + 'search_workspace_symbols', + 'list_code_usages', + 'get_errors', + 'get_vscode_api', + 'get_changed_files', + 'create_new_workspace', + 'install_extension', + 'get_project_setup_info', + 'create_and_run_task', + 'run_task', + 'get_task_output', + 'run_vscode_command', + 'multi_replace_string_in_file', + 'install_python_packages', + 'get_search_view_results', + 'vscode_searchExtensions_internal', + 'read_project_structure' + ] + }, + 'Testing': { + summary: 'Call tools from this group when you need to run tests, analyze test failures, and manage test workflows.', + tools: [ + 'run_tests', + 'test_failure', + 'test_search', + 'runTests' + ] + }, + 'Redundant but Specific': { + summary: 'These tools have overlapping functionalities but are highly specialized for certain tasks. \nTools: file_search, get_terminal_selection, get_terminal_last_command, create_directory, get_doc_info', + tools: [ + 'file_search', + 'get_terminal_selection', + 'get_terminal_last_command', + 'create_directory', + 'get_doc_info' + ] + } +} as const; + +export class BuiltInToolGroupHandler { + constructor( + private readonly _telemetryService: ITelemetryService, + ) { } + + /** Creates groups for built-in tools based on the pre-defined enum above*/ + createBuiltInToolGroups(tools: LanguageModelToolInformation[]): (VirtualTool | LanguageModelToolInformation)[] { + // If there are too few tools, don't group them + if (tools.length <= Constant.MIN_TOOLSET_SIZE_TO_GROUP) { + return tools; + } + + // Create a map of tool names to tool objects for easy lookup + const toolMap = new Map(tools.map(tool => [tool.name, tool])); + + const virtualTools: VirtualTool[] = []; + const usedTools = new Set(); + + // Create virtual tools for each predefined group + for (const [groupName, groupDef] of Object.entries(BUILT_IN_TOOL_GROUPS)) { + const groupTools = groupDef.tools + .map(toolName => toolMap.get(toolName)) + .filter((tool): tool is LanguageModelToolInformation => tool !== undefined); + + if (groupTools.length > 0) { + // Mark each tool that has already been added to a group + groupTools.forEach(tool => usedTools.add(tool.name)); + + // Create the virtual tool group + const virtualTool = new VirtualTool( + VIRTUAL_TOOL_NAME_PREFIX + groupName.toLowerCase().replace(/\s+/g, '_'), + SUMMARY_PREFIX + groupDef.summary + SUMMARY_SUFFIX, + 0, + { + toolsetKey: BUILT_IN_GROUP, + groups: [], + possiblePrefix: 'builtin_' + }, + groupTools + ); + virtualTools.push(virtualTool); + } + } + + // Add any remaining uncategorized tools individually + const uncategorizedTools = tools.filter(tool => !usedTools.has(tool.name)); + + // Send telemetry for built-in tool grouping + this._telemetryService.sendMSFTTelemetryEvent('virtualTools.generate', { + groupKey: BUILT_IN_GROUP, + }, { + uncategorized: uncategorizedTools.length, + toolsBefore: tools.length, + toolsAfter: virtualTools.length, + retries: 0, // No retries for predefined groups + durationMs: 0, // Instant for predefined groups + }); + + this._telemetryService.sendEnhancedGHTelemetryEvent('virtualTools.generate', { + groupKey: BUILT_IN_GROUP, + }, { + uncategorized: uncategorizedTools.length, + toolsBefore: tools.length, + toolsAfter: virtualTools.length, + retries: 0, // No retries for predefined groups + durationMs: 0, // Instant for predefined groups + }); + + return [...virtualTools, ...uncategorizedTools]; + } + + static get BUILT_IN_GROUP_KEY(): string { + return BUILT_IN_GROUP; + } +} \ No newline at end of file diff --git a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts index 70a7306d8..2f10aec8f 100644 --- a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts +++ b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts @@ -17,90 +17,20 @@ import { Iterable } from '../../../../util/vs/base/common/iterator'; import { StopWatch } from '../../../../util/vs/base/common/stopwatch'; import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation'; import { LanguageModelToolExtensionSource, LanguageModelToolMCPSource } from '../../../../vscodeTypes'; +import { BuiltInToolGroupHandler } from './builtInToolGroupHandler'; import { EMBEDDING_TYPE_FOR_TOOL_GROUPING, ToolEmbeddingsComputer } from './toolEmbeddingsCache'; import { VIRTUAL_TOOL_NAME_PREFIX, VirtualTool } from './virtualTool'; import { divideToolsIntoExistingGroups, divideToolsIntoGroups, summarizeToolGroup } from './virtualToolSummarizer'; import { ISummarizedToolCategory, IToolCategorization, IToolGroupingCache } from './virtualToolTypes'; import * as Constant from './virtualToolsConstants'; -const BUILT_IN_GROUP = 'builtin'; const CATEGORIZATION_ENDPOINT = CHAT_MODEL.GPT4OMINI; const SUMMARY_PREFIX = 'Call this tool when you need access to a new category of tools. The category of tools is described as follows:\n\n'; const SUMMARY_SUFFIX = '\n\nBe sure to call this tool if you need a capability related to the above.'; -// categorize all tools except for the 11 default tools -// 12 default tools = semantic_search, grep_search, read_file, create_file, apply_patch, replace_string_in_file, -// insert_edit_into_file, run_in_terminal, list_dir, think, get_terminal_output, manage_todo_list -const BUILT_IN_TOOL_GROUPS = { - 'Jupyter Notebook Tools': { - summary: 'Call tools from this group when you need to work with Jupyter notebooks - creating, editing, running cells, and managing notebook operations.', - tools: [ - 'create_new_jupyter_notebook', - 'edit_notebook_file', - 'run_notebook_cell', - 'copilot_getNotebookSummary', - 'read_notebook_cell_output', - 'configure_notebook', - 'notebook_install_packages', - 'notebook_list_packages', - 'configure_python_environment', - 'get_python_environment_details', - 'get_python_executable_details' - ] - }, - 'Web Interaction': { - summary: 'Call tools from this group when you need to interact with web content, browse websites, or access external resources.', - tools: [ - 'fetch_webpage', - 'open_simple_browser', - 'github_repo' - ] - }, - 'VS Code Interaction': { - summary: 'Call tools from this group when you need to interact with the VS Code workspace and access VS Code features.', - tools: [ - 'search_workspace_symbols', - 'list_code_usages', - 'get_errors', - 'get_vscode_api', - 'get_changed_files', - 'create_new_workspace', - 'install_extension', - 'get_project_setup_info', - 'create_and_run_task', - 'run_task', - 'get_task_output', - 'run_vscode_command', - 'multi_replace_string_in_file', - 'install_python_packages', - 'get_search_view_results', - 'vscode_searchExtensions_internal', - 'read_project_structure' - ] - }, - 'Testing': { - summary: 'Call tools from this group when you need to run tests, analyze test failures, and manage test workflows.', - tools: [ - 'run_tests', - 'test_failure', - 'test_search', - 'runTests' - ] - }, - 'Redundant but Specific': { - summary: 'These tools have overlapping functionalities but are highly specialized for certain tasks. \nTools: file_search, get_terminal_selection, get_terminal_last_command, create_directory, get_doc_info', - tools: [ - 'file_search', - 'get_terminal_selection', - 'get_terminal_last_command', - 'create_directory', - 'get_doc_info' - ] - } -} as const; - export class VirtualToolGrouper implements IToolCategorization { private readonly toolEmbeddingsComputer: ToolEmbeddingsComputer; + private readonly builtInToolGroupHandler: BuiltInToolGroupHandler; constructor( @IEndpointProvider private readonly _endpointProvider: IEndpointProvider, @@ -113,6 +43,7 @@ export class VirtualToolGrouper implements IToolCategorization { @IInstantiationService _instantiationService: IInstantiationService, ) { this.toolEmbeddingsComputer = _instantiationService.createInstance(ToolEmbeddingsComputer); + this.builtInToolGroupHandler = new BuiltInToolGroupHandler(_telemetryService); } async addGroups(query: string, root: VirtualTool, tools: LanguageModelToolInformation[], token: CancellationToken): Promise { @@ -128,7 +59,7 @@ export class VirtualToolGrouper implements IToolCategorization { } else if (t.source instanceof LanguageModelToolMCPSource) { return 'mcp_' + t.source.label; } else { - return BUILT_IN_GROUP; + return BuiltInToolGroupHandler.BUILT_IN_GROUP_KEY; } }); @@ -295,10 +226,19 @@ export class VirtualToolGrouper implements IToolCategorization { /** Top-level request to categorize a group of tools from a single source. */ private async _generateGroupsFromToolset(key: string, tools: LanguageModelToolInformation[], previous: ISummarizedToolCategory[] | undefined, token: CancellationToken): Promise<(VirtualTool | LanguageModelToolInformation)[]> { // Handle built-in tools with predefined groups only if experimental setting is enabled - if (key === BUILT_IN_GROUP) { + if (key === BuiltInToolGroupHandler.BUILT_IN_GROUP_KEY) { const defaultToolGroupingEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.DefaultToolsGrouped, this._expService); if (defaultToolGroupingEnabled) { - return this._createBuiltInToolGroups(tools); + // Get the model family from the categorization endpoint to check if grouping should apply + const endpoint = await this._endpointProvider.getChatEndpoint(CATEGORIZATION_ENDPOINT); + const modelFamily = endpoint?.family; + + // Only apply grouping for GPT-4.1 or GPT-5 models + if (!modelFamily || (modelFamily !== 'gpt-4.1' && !modelFamily.startsWith('gpt-5'))) { + return tools; + } + + return this.builtInToolGroupHandler.createBuiltInToolGroups(tools); } else { return tools; } @@ -379,61 +319,6 @@ export class VirtualToolGrouper implements IToolCategorization { return virtualTools.concat(uncategorized); } - /** Creates groups for built-in tools based on the pre-defined enum above*/ - private _createBuiltInToolGroups(tools: LanguageModelToolInformation[]): (VirtualTool | LanguageModelToolInformation)[] { - // If there are too few tools, don't group them - if (tools.length <= Constant.MIN_TOOLSET_SIZE_TO_GROUP) { - return tools; - } - - // Create a map of tool names to tool objects for easy lookup - const toolMap = new Map(tools.map(tool => [tool.name, tool])); - - const virtualTools: VirtualTool[] = []; - const usedTools = new Set(); - - // Create virtual tools for each predefined group - for (const [groupName, groupDef] of Object.entries(BUILT_IN_TOOL_GROUPS)) { - const groupTools = groupDef.tools - .map(toolName => toolMap.get(toolName)) - .filter((tool): tool is LanguageModelToolInformation => tool !== undefined); - - if (groupTools.length > 0) { - // Mark each tool that has already been added to a group - groupTools.forEach(tool => usedTools.add(tool.name)); - - // Create the virtual tool group - const virtualTool = new VirtualTool( - VIRTUAL_TOOL_NAME_PREFIX + groupName.toLowerCase().replace(/\s+/g, '_'), - SUMMARY_PREFIX + groupDef.summary + SUMMARY_SUFFIX, - 0, - { - toolsetKey: BUILT_IN_GROUP, - groups: [], - possiblePrefix: 'builtin_' - }, - groupTools - ); - virtualTools.push(virtualTool); - } - } - - // Add any remaining uncategorized tools individually - const uncategorizedTools = tools.filter(tool => !usedTools.has(tool.name)); - - // Send telemetry for built-in tool grouping - this._telemetryService.sendMSFTTelemetryEvent('virtualTools.generate', { - groupKey: BUILT_IN_GROUP, - }, { - uncategorized: uncategorizedTools.length, - toolsBefore: tools.length, - toolsAfter: virtualTools.length, - retries: 0, // No retries for predefined groups - durationMs: 0, // Instant for predefined groups - }); - - return [...virtualTools, ...uncategorizedTools]; - } private async _getPredictedTools(query: string, tools: LanguageModelToolInformation[], token: CancellationToken): Promise { // compute the embeddings for the query From 9b2a91fa34f29e09270bdfeb682345ce4ceb698b Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Thu, 25 Sep 2025 11:54:36 -0700 Subject: [PATCH 06/10] remove unnecessary telemetry sending --- .../common/virtualTools/builtInToolGroupHandler.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts b/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts index e3b262862..c726d2d99 100644 --- a/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts +++ b/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts @@ -95,7 +95,6 @@ export class BuiltInToolGroupHandler { return tools; } - // Create a map of tool names to tool objects for easy lookup const toolMap = new Map(tools.map(tool => [tool.name, tool])); const virtualTools: VirtualTool[] = []; @@ -111,7 +110,6 @@ export class BuiltInToolGroupHandler { // Mark each tool that has already been added to a group groupTools.forEach(tool => usedTools.add(tool.name)); - // Create the virtual tool group const virtualTool = new VirtualTool( VIRTUAL_TOOL_NAME_PREFIX + groupName.toLowerCase().replace(/\s+/g, '_'), SUMMARY_PREFIX + groupDef.summary + SUMMARY_SUFFIX, @@ -141,16 +139,6 @@ export class BuiltInToolGroupHandler { durationMs: 0, // Instant for predefined groups }); - this._telemetryService.sendEnhancedGHTelemetryEvent('virtualTools.generate', { - groupKey: BUILT_IN_GROUP, - }, { - uncategorized: uncategorizedTools.length, - toolsBefore: tools.length, - toolsAfter: virtualTools.length, - retries: 0, // No retries for predefined groups - durationMs: 0, // Instant for predefined groups - }); - return [...virtualTools, ...uncategorizedTools]; } From f66be55442299fc2bec337198320b9393a9549fa Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Thu, 2 Oct 2025 10:07:16 -0700 Subject: [PATCH 07/10] adding intermediate changes --- package.json | 4 +-- .../virtualTools/builtInToolGroupHandler.ts | 9 +++--- .../common/virtualTools/virtualToolGrouper.ts | 29 +++++++++++++++---- .../common/configurationService.ts | 2 +- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 80facda87..9f87f1d10 100644 --- a/package.json +++ b/package.json @@ -2895,8 +2895,8 @@ "github.copilot.chat.virtualTools.threshold": { "type": "number", "minimum": 0, - "maximum": 24, - "default": 24, + "maximum": 28, + "default": 28, "tags": [ "experimental" ], diff --git a/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts b/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts index c726d2d99..6f238142b 100644 --- a/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts +++ b/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts @@ -12,7 +12,7 @@ const BUILT_IN_GROUP = 'builtin'; const SUMMARY_PREFIX = 'Call this tool when you need access to a new category of tools. The category of tools is described as follows:\n\n'; const SUMMARY_SUFFIX = '\n\nBe sure to call this tool if you need a capability related to the above.'; -// categorize all tools except for the 11 default tools +// categorize all tools except for the 12 default tools // 12 default tools = semantic_search, grep_search, read_file, create_file, apply_patch, replace_string_in_file, // insert_edit_into_file, run_in_terminal, list_dir, think, get_terminal_output, manage_todo_list const BUILT_IN_TOOL_GROUPS = { @@ -55,7 +55,6 @@ const BUILT_IN_TOOL_GROUPS = { 'run_task', 'get_task_output', 'run_vscode_command', - 'multi_replace_string_in_file', 'install_python_packages', 'get_search_view_results', 'vscode_searchExtensions_internal', @@ -72,13 +71,15 @@ const BUILT_IN_TOOL_GROUPS = { ] }, 'Redundant but Specific': { - summary: 'These tools have overlapping functionalities but are highly specialized for certain tasks. \nTools: file_search, get_terminal_selection, get_terminal_last_command, create_directory, get_doc_info', + summary: 'These tools have overlapping functionalities but are highly specialized for certain tasks. \nTools: file_search, get_terminal_selection, get_terminal_last_command, create_directory, get_doc_info, multi_replace_string_in_file, edit_files', tools: [ 'file_search', 'get_terminal_selection', 'get_terminal_last_command', 'create_directory', - 'get_doc_info' + 'get_doc_info', + 'edit_files', + 'multi_replace_string_in_file' ] } } as const; diff --git a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts index 2f10aec8f..a84fa53d8 100644 --- a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts +++ b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts @@ -31,6 +31,7 @@ const SUMMARY_SUFFIX = '\n\nBe sure to call this tool if you need a capability r export class VirtualToolGrouper implements IToolCategorization { private readonly toolEmbeddingsComputer: ToolEmbeddingsComputer; private readonly builtInToolGroupHandler: BuiltInToolGroupHandler; + private _hasGroupedDefaultTools = false; constructor( @IEndpointProvider private readonly _endpointProvider: IEndpointProvider, @@ -48,11 +49,13 @@ export class VirtualToolGrouper implements IToolCategorization { async addGroups(query: string, root: VirtualTool, tools: LanguageModelToolInformation[], token: CancellationToken): Promise { // If there's no need to group tools, just add them all directly; + // TODO use Constant.START_GROUPING_AFTER_TOOL_COUNT to be dynamic based on model? or can I use this logic for every model? if (tools.length < Constant.START_GROUPING_AFTER_TOOL_COUNT) { root.contents = tools; return; } + // TODO add logic here to group default tools for gpt models const byToolset = groupBy(tools, t => { if (t.source instanceof LanguageModelToolExtensionSource) { return 'ext_' + t.source.id; @@ -202,8 +205,18 @@ export class VirtualToolGrouper implements IToolCategorization { } // Get unexpanded virtual tools, sorted by the ranker function (ascending order). + // If we've grouped default tools, exclude built-in tool groups from expansion const expandable = root.contents - .filter((t): t is VirtualTool => t instanceof VirtualTool && !t.isExpanded) + .filter((t): t is VirtualTool => { + if (!(t instanceof VirtualTool) || t.isExpanded) { + return false; + } + // Skip built-in tool groups if we've grouped default tools for GPT models + if (this._hasGroupedDefaultTools && t.metadata.toolsetKey === BuiltInToolGroupHandler.BUILT_IN_GROUP_KEY) { + return false; + } + return true; + }) .sort((a, b) => ranker(a) - ranker(b)); // Expand them until we hit the target limit @@ -229,15 +242,19 @@ export class VirtualToolGrouper implements IToolCategorization { if (key === BuiltInToolGroupHandler.BUILT_IN_GROUP_KEY) { const defaultToolGroupingEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.DefaultToolsGrouped, this._expService); if (defaultToolGroupingEnabled) { + // TODO fix this logic to work for when the chat model is `gpt-4.1` or `gpt-5` // Get the model family from the categorization endpoint to check if grouping should apply - const endpoint = await this._endpointProvider.getChatEndpoint(CATEGORIZATION_ENDPOINT); - const modelFamily = endpoint?.family; + // const endpoint = await this._endpointProvider.getChatEndpoint(); + // const modelFamily = endpoint?.family; // Only apply grouping for GPT-4.1 or GPT-5 models - if (!modelFamily || (modelFamily !== 'gpt-4.1' && !modelFamily.startsWith('gpt-5'))) { - return tools; - } + // For Sonnet models (claude), do not group default tools - let them expand to the 128 limit + // if (!modelFamily || (!modelFamily.includes('gpt-4.1') && !modelFamily.startsWith('gpt-5') && !modelFamily.startsWith('o1') && !modelFamily.startsWith('o3'))) { + // return tools; + // } + // Mark that we've grouped default tools so we don't aggressively expand later + this._hasGroupedDefaultTools = true; return this.builtInToolGroupHandler.createBuiltInToolGroups(tools); } else { return tools; diff --git a/src/platform/configuration/common/configurationService.ts b/src/platform/configuration/common/configurationService.ts index c4729df7d..00a905fb9 100644 --- a/src/platform/configuration/common/configurationService.ts +++ b/src/platform/configuration/common/configurationService.ts @@ -522,7 +522,7 @@ export function defineExpSetting(key: strin } // Max CAPI tool count limit -export const HARD_TOOL_LIMIT = 24; +export const HARD_TOOL_LIMIT = 28; // WARNING // These values are used in the request and are case sensitive. Do not change them unless advised by CAPI. From 6741dc8a8de8ecbb0119fd29591dae95dec77a8d Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Thu, 2 Oct 2025 19:47:24 -0700 Subject: [PATCH 08/10] updated explicit tool set --- .../tools/common/virtualTools/builtInToolGroupHandler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts b/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts index 6f238142b..b36616599 100644 --- a/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts +++ b/src/extension/tools/common/virtualTools/builtInToolGroupHandler.ts @@ -71,11 +71,11 @@ const BUILT_IN_TOOL_GROUPS = { ] }, 'Redundant but Specific': { - summary: 'These tools have overlapping functionalities but are highly specialized for certain tasks. \nTools: file_search, get_terminal_selection, get_terminal_last_command, create_directory, get_doc_info, multi_replace_string_in_file, edit_files', + summary: 'These tools have overlapping functionalities but are highly specialized for certain tasks. \nTools: file_search, terminal_selection, terminal_last_command, create_directory, get_doc_info, multi_replace_string_in_file, edit_files', tools: [ 'file_search', - 'get_terminal_selection', - 'get_terminal_last_command', + 'terminal_selection', + 'terminal_last_command', 'create_directory', 'get_doc_info', 'edit_files', From 20317176baea55b95a93d7370a7b253560a3a371 Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Mon, 6 Oct 2025 11:33:36 -0700 Subject: [PATCH 09/10] only perform grouping for gpt 4.1 and 5 --- .../node/defaultIntentRequestHandler.ts | 4 +-- .../tools/common/virtualTools/toolGrouping.ts | 13 ++++---- .../common/virtualTools/virtualToolGrouper.ts | 30 +++++++++++-------- .../common/virtualTools/virtualToolTypes.ts | 7 +++-- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/extension/prompt/node/defaultIntentRequestHandler.ts b/src/extension/prompt/node/defaultIntentRequestHandler.ts index 53c365a1e..5605e83c9 100644 --- a/src/extension/prompt/node/defaultIntentRequestHandler.ts +++ b/src/extension/prompt/node/defaultIntentRequestHandler.ts @@ -595,7 +595,7 @@ class DefaultToolCallingLoop extends ToolCallingLoop { const token = CancellationToken.None; const allTools = await this.options.invocation.getAvailableTools?.() ?? []; const grouping = this.toolGroupingService.create(this.options.conversation.sessionId, allTools); - const computed = await grouping.compute(this.options.request.prompt, token); + const computed = await grouping.compute(this.options.request.prompt, token, this.options.invocation.endpoint); const container = grouping.getContainerFor(candidateCall.name); @@ -722,7 +722,7 @@ class DefaultToolCallingLoop extends ToolCallingLoop { return tools; } - const computePromise = this.toolGrouping.compute(this.options.request.prompt, token); + const computePromise = this.toolGrouping.compute(this.options.request.prompt, token, this.options.invocation.endpoint); // Show progress if this takes a moment... const timeout = setTimeout(() => { diff --git a/src/extension/tools/common/virtualTools/toolGrouping.ts b/src/extension/tools/common/virtualTools/toolGrouping.ts index d0d920801..972ad2b76 100644 --- a/src/extension/tools/common/virtualTools/toolGrouping.ts +++ b/src/extension/tools/common/virtualTools/toolGrouping.ts @@ -5,6 +5,7 @@ import type { LanguageModelToolInformation } from 'vscode'; import { ConfigKey, HARD_TOOL_LIMIT, IConfigurationService } from '../../../../platform/configuration/common/configurationService'; +import { IChatEndpoint } from '../../../../platform/networking/common/networking'; import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry'; import { equals as arraysEqual, uniqueFilter } from '../../../../util/vs/base/common/arrays'; @@ -126,19 +127,19 @@ export class ToolGrouping implements IToolGrouping { this._expandOnNext.add(toolName); } - async compute(query: string, token: CancellationToken): Promise { - await this._doCompute(query, token); + async compute(query: string, token: CancellationToken, endpoint?: IChatEndpoint): Promise { + await this._doCompute(query, token, endpoint); return [...this._root.tools()].filter(uniqueFilter(t => t.name)); } - async computeAll(query: string, token: CancellationToken): Promise<(LanguageModelToolInformation | VirtualTool)[]> { - await this._doCompute(query, token); + async computeAll(query: string, token: CancellationToken, endpoint?: IChatEndpoint): Promise<(LanguageModelToolInformation | VirtualTool)[]> { + await this._doCompute(query, token, endpoint); return this._root.contents; } - private async _doCompute(query: string, token: CancellationToken) { + private async _doCompute(query: string, token: CancellationToken, endpoint?: IChatEndpoint) { if (this._didToolsChange) { - await this._grouper.addGroups(query, this._root, this._tools.slice(), token); + await this._grouper.addGroups(query, this._root, this._tools.slice(), token, endpoint); this._didToolsChange = false; } diff --git a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts index a8c04b291..4e5e0f055 100644 --- a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts +++ b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts @@ -8,6 +8,7 @@ import { CHAT_MODEL, ConfigKey, HARD_TOOL_LIMIT, IConfigurationService } from '. import { IEmbeddingsComputer } from '../../../../platform/embeddings/common/embeddingsComputer'; import { IEndpointProvider } from '../../../../platform/endpoint/common/endpointProvider'; import { ILogService } from '../../../../platform/log/common/logService'; +import { IChatEndpoint } from '../../../../platform/networking/common/networking'; import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry'; import { TelemetryCorrelationId } from '../../../../util/common/telemetryCorrelationId'; @@ -51,7 +52,7 @@ export class VirtualToolGrouper implements IToolCategorization { return this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.VirtualToolEmbeddingRanking, this._expService); } - async addGroups(query: string, root: VirtualTool, tools: LanguageModelToolInformation[], token: CancellationToken): Promise { + async addGroups(query: string, root: VirtualTool, tools: LanguageModelToolInformation[], token: CancellationToken, endpoint?: IChatEndpoint): Promise { // If there's no need to group tools, just add them all directly; // TODO use Constant.START_GROUPING_AFTER_TOOL_COUNT to be dynamic based on model? or can I use this logic for every model? if (tools.length < Constant.START_GROUPING_AFTER_TOOL_COUNT) { @@ -86,7 +87,7 @@ export class VirtualToolGrouper implements IToolCategorization { // Now all toolsets (including built-in) go through _generateGroupsFromToolset const grouped = await Promise.all(Object.entries(byToolset).map(([key, tools]) => { - return this._generateGroupsFromToolset(key, tools, previousCategorizations.get(key), token); + return this._generateGroupsFromToolset(key, tools, previousCategorizations.get(key), token, endpoint); })); this._cache.flush(); @@ -249,21 +250,24 @@ export class VirtualToolGrouper implements IToolCategorization { } /** Top-level request to categorize a group of tools from a single source. */ - private async _generateGroupsFromToolset(key: string, tools: LanguageModelToolInformation[], previous: ISummarizedToolCategory[] | undefined, token: CancellationToken): Promise<(VirtualTool | LanguageModelToolInformation)[]> { + private async _generateGroupsFromToolset(key: string, tools: LanguageModelToolInformation[], previous: ISummarizedToolCategory[] | undefined, token: CancellationToken, endpoint?: IChatEndpoint): Promise<(VirtualTool | LanguageModelToolInformation)[]> { // Handle built-in tools with predefined groups only if experimental setting is enabled if (key === BuiltInToolGroupHandler.BUILT_IN_GROUP_KEY) { const defaultToolGroupingEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.DefaultToolsGrouped, this._expService); if (defaultToolGroupingEnabled) { - // TODO fix this logic to work for when the chat model is `gpt-4.1` or `gpt-5` - // Get the model family from the categorization endpoint to check if grouping should apply - // const endpoint = await this._endpointProvider.getChatEndpoint(); - // const modelFamily = endpoint?.family; - - // Only apply grouping for GPT-4.1 or GPT-5 models - // For Sonnet models (claude), do not group default tools - let them expand to the 128 limit - // if (!modelFamily || (!modelFamily.includes('gpt-4.1') && !modelFamily.startsWith('gpt-5') && !modelFamily.startsWith('o1') && !modelFamily.startsWith('o3'))) { - // return tools; - // } + + // Get the model family from the current chat endpoint to check if grouping should apply + // Use the passed endpoint if available, otherwise fallback to default endpoint + const currentEndpoint = endpoint ?? (await this._endpointProvider.getAllChatEndpoints()).find(e => e.isDefault) ?? await this._endpointProvider.getChatEndpoint('gpt-4.1'); + const modelFamily = currentEndpoint?.family; + + // Only apply grouping for GPT-4.1 or GPT-5 + // For other models (like Claude/Sonnet), do not group default tools - let them expand to the limit + if (!modelFamily || + !(modelFamily === 'gpt-4.1' || + modelFamily.startsWith('gpt-5'))) { + return tools; + } // Mark that we've grouped default tools so we don't aggressively expand later this._hasGroupedDefaultTools = true; diff --git a/src/extension/tools/common/virtualTools/virtualToolTypes.ts b/src/extension/tools/common/virtualTools/virtualToolTypes.ts index 788a12649..880eaef8e 100644 --- a/src/extension/tools/common/virtualTools/virtualToolTypes.ts +++ b/src/extension/tools/common/virtualTools/virtualToolTypes.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import type { LanguageModelToolInformation, LanguageModelToolResult } from 'vscode'; +import { IChatEndpoint } from '../../../../platform/networking/common/networking'; import { createServiceIdentifier } from '../../../../util/common/services'; import { CancellationToken } from '../../../../util/vs/base/common/cancellation'; import { IObservable } from '../../../../util/vs/base/common/observableInternal'; @@ -53,12 +54,12 @@ export interface IToolGrouping { * Returns a list of tools that should be used for the given request. * Internally re-reads the request and conversation state. */ - compute(query: string, token: CancellationToken): Promise; + compute(query: string, token: CancellationToken, endpoint?: IChatEndpoint): Promise; /** * Returns the complete tree of tools, used for diagnostic purposes. */ - computeAll(query: string, token: CancellationToken): Promise<(LanguageModelToolInformation | VirtualTool)[]>; + computeAll(query: string, token: CancellationToken, endpoint?: IChatEndpoint): Promise<(LanguageModelToolInformation | VirtualTool)[]>; } export interface IToolGroupingService { @@ -103,7 +104,7 @@ export interface IToolCategorization { * Called whenever new tools are added. The function should add each tool into * the appropriate virtual tool or top-level tool in the `root`. */ - addGroups(query: string, root: VirtualTool, tools: LanguageModelToolInformation[], token: CancellationToken): Promise; + addGroups(query: string, root: VirtualTool, tools: LanguageModelToolInformation[], token: CancellationToken, endpoint?: IChatEndpoint): Promise; /** * Recalculates the "embeddings" group, when enabled, so relevant tools From 74b7a7ad4e0e1d8fc98dd47d43032af1dbce3397 Mon Sep 17 00:00:00 2001 From: Anisha Agarwal Date: Mon, 6 Oct 2025 19:15:43 -0700 Subject: [PATCH 10/10] updated the hard tool limit back to 128, added logic --- package.json | 4 ++-- .../tools/common/virtualTools/toolGrouping.ts | 8 +++++++- .../common/virtualTools/virtualToolGrouper.ts | 14 +++++++++++--- .../common/virtualTools/virtualToolsConstants.ts | 2 ++ .../configuration/common/configurationService.ts | 2 +- 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 5444394b9..b4557f9dd 100644 --- a/package.json +++ b/package.json @@ -2911,8 +2911,8 @@ "github.copilot.chat.virtualTools.threshold": { "type": "number", "minimum": 0, - "maximum": 28, - "default": 28, + "maximum": 128, + "default": 128, "tags": [ "experimental" ], diff --git a/src/extension/tools/common/virtualTools/toolGrouping.ts b/src/extension/tools/common/virtualTools/toolGrouping.ts index 972ad2b76..8725bf201 100644 --- a/src/extension/tools/common/virtualTools/toolGrouping.ts +++ b/src/extension/tools/common/virtualTools/toolGrouping.ts @@ -48,7 +48,13 @@ export class ToolGrouping implements IToolGrouping { } public get isEnabled() { - return this._tools.length >= computeToolGroupingMinThreshold(this._experimentationService, this._configurationService).get(); + // Match the logic from VirtualToolGrouper.addGroups() + // Enable if we could potentially trigger built-in grouping (when GPT model is used) + const defaultToolGroupingEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.DefaultToolsGrouped, this._experimentationService); + const couldTriggerBuiltInGrouping = this._tools.length > Constant.START_BUILTIN_GROUPING_AFTER_TOOL_COUNT && defaultToolGroupingEnabled; + + // Or if we meet the standard threshold for all tool types + return couldTriggerBuiltInGrouping || this._tools.length >= Constant.START_GROUPING_AFTER_TOOL_COUNT; } constructor( diff --git a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts index 4e5e0f055..78e32d00c 100644 --- a/src/extension/tools/common/virtualTools/virtualToolGrouper.ts +++ b/src/extension/tools/common/virtualTools/virtualToolGrouper.ts @@ -54,13 +54,21 @@ export class VirtualToolGrouper implements IToolCategorization { async addGroups(query: string, root: VirtualTool, tools: LanguageModelToolInformation[], token: CancellationToken, endpoint?: IChatEndpoint): Promise { // If there's no need to group tools, just add them all directly; - // TODO use Constant.START_GROUPING_AFTER_TOOL_COUNT to be dynamic based on model? or can I use this logic for every model? - if (tools.length < Constant.START_GROUPING_AFTER_TOOL_COUNT) { + + // if the model is gpt 4.1 or gpt-5 and there are more than START_BUILTIN_GROUPING_AFTER_TOOL_COUNT tools, we should group built-in tools + // otherwise, follow the existing logic of grouping all tools together + const currentEndpoint = endpoint ?? (await this._endpointProvider.getAllChatEndpoints()).find(e => e.isDefault) ?? await this._endpointProvider.getChatEndpoint('gpt-4.1'); + const modelFamily = currentEndpoint?.family; + const isGpt = modelFamily?.startsWith('gpt-4.1') || modelFamily?.startsWith('gpt-5'); + const defaultToolGroupingEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.Internal.DefaultToolsGrouped, this._expService); + + const triggerBuiltInGrouping = isGpt && tools.length > Constant.START_BUILTIN_GROUPING_AFTER_TOOL_COUNT && defaultToolGroupingEnabled; + + if (!triggerBuiltInGrouping && tools.length < Constant.START_GROUPING_AFTER_TOOL_COUNT) { root.contents = tools; return; } - // TODO add logic here to group default tools for gpt models const byToolset = groupBy(tools, t => { if (t.source instanceof LanguageModelToolExtensionSource) { return 'ext_' + t.source.id; diff --git a/src/extension/tools/common/virtualTools/virtualToolsConstants.ts b/src/extension/tools/common/virtualTools/virtualToolsConstants.ts index 6e7716367..a00b46849 100644 --- a/src/extension/tools/common/virtualTools/virtualToolsConstants.ts +++ b/src/extension/tools/common/virtualTools/virtualToolsConstants.ts @@ -8,6 +8,8 @@ import { HARD_TOOL_LIMIT } from '../../../../platform/configuration/common/confi /** Point after which we'll start grouping tools */ export const START_GROUPING_AFTER_TOOL_COUNT = HARD_TOOL_LIMIT / 2; // 64, currently +export const START_BUILTIN_GROUPING_AFTER_TOOL_COUNT = 20; // Lower bound above which we trigger built-in tool grouping + /** Re-expand groups until we have at least this many tools. */ export const EXPAND_UNTIL_COUNT = START_GROUPING_AFTER_TOOL_COUNT; /** diff --git a/src/platform/configuration/common/configurationService.ts b/src/platform/configuration/common/configurationService.ts index feca817f2..74f920c3c 100644 --- a/src/platform/configuration/common/configurationService.ts +++ b/src/platform/configuration/common/configurationService.ts @@ -522,7 +522,7 @@ export function defineExpSetting(key: strin } // Max CAPI tool count limit -export const HARD_TOOL_LIMIT = 28; +export const HARD_TOOL_LIMIT = 128; // WARNING // These values are used in the request and are case sensitive. Do not change them unless advised by CAPI.