Make agent stests optional (#514)

* Make agent stests optional

* And this

* Marked as optional

Author: roblourens

test/e2e/edit.stest.ts

@@ -11,9 +11,10 @@ import { IReadFileParamsV1 } from '../../src/extension/tools/node/readFileTool';
 import { deserializeWorkbenchState } from '../../src/platform/test/node/promptContextModel';
 import { ssuite, stest } from '../base/stest';
 import { generateToolTestRunner } from './toolSimTest';
+import { shouldSkipAgentTests } from './tools.stest';
 
 
-ssuite({ title: 'edit', subtitle: 'toolCalling', location: 'panel' }, () => {
+ssuite.optional(shouldSkipAgentTests, { title: 'edit', subtitle: 'toolCalling', location: 'panel' }, () => {
 	const scenarioFolder = path.join(__dirname, '..', 'test/scenarios/test-tools');
 	const getState = () => deserializeWorkbenchState(scenarioFolder, path.join(scenarioFolder, 'chatSetup.state.json'));
 

test/e2e/fetchWebPageTool.stest.ts

@@ -9,13 +9,14 @@ import { ToolName } from '../../src/extension/tools/common/toolNames';
 import { deserializeWorkbenchState } from '../../src/platform/test/node/promptContextModel';
 import { ssuite, stest } from '../base/stest';
 import { generateToolTestRunner } from './toolSimTest';
+import { shouldSkipAgentTests } from './tools.stest';
 
 interface IFetchWebPageToolParams {
 	urls: string[];
 	query?: string;
 }
 
-ssuite({ title: 'fetchWebPageTool', subtitle: 'toolCalling', location: 'panel' }, () => {
+ssuite.optional(shouldSkipAgentTests, { title: 'fetchWebPageTool', subtitle: 'toolCalling', location: 'panel' }, () => {
 	const scenarioFolder = path.join(__dirname, '..', 'test/scenarios/test-tools');
 	const getState = () => deserializeWorkbenchState(scenarioFolder, path.join(scenarioFolder, 'tools.state.json'));
 

test/e2e/findFilesTool.stest.ts

@@ -10,8 +10,9 @@ import { IFindFilesToolParams } from '../../src/extension/tools/node/findFilesTo
 import { deserializeWorkbenchState } from '../../src/platform/test/node/promptContextModel';
 import { ssuite, stest } from '../base/stest';
 import { generateToolTestRunner } from './toolSimTest';
+import { shouldSkipAgentTests } from './tools.stest';
 
-ssuite({ title: 'findFilesTool', subtitle: 'toolCalling', location: 'panel' }, () => {
+ssuite.optional(shouldSkipAgentTests, { title: 'findFilesTool', subtitle: 'toolCalling', location: 'panel' }, () => {
 	const scenarioFolder = path.join(__dirname, '..', 'test/scenarios/test-tools');
 	const getState = () => deserializeWorkbenchState(scenarioFolder, path.join(scenarioFolder, 'tools.state.json'));
 

test/e2e/notebookTools.stest.ts

@@ -10,8 +10,9 @@ import { getCellId } from '../../src/platform/notebook/common/helpers';
 import { deserializeWorkbenchState } from '../../src/platform/test/node/promptContextModel';
 import { ssuite, stest } from '../base/stest';
 import { generateToolTestRunner } from './toolSimTest';
+import { shouldSkipAgentTests } from './tools.stest';
 
-ssuite({
+ssuite.optional(shouldSkipAgentTests, {
 	title: 'notebooks', subtitle: 'toolCalling', location: 'panel', configurations: []
 }, (inputPath) => {
 	const scenarioFolder = inputPath ?? path.join(__dirname, '..', 'test/scenarios/test-notebook-tools');

test/e2e/tools.stest.ts

@@ -8,7 +8,11 @@ import path from 'path';
 import { ssuite, stest } from '../base/stest';
 import { fetchToolScenarios, generateToolTestRunner } from './toolSimTest';
 
-ssuite({ title: 'toolCalling', location: 'panel' }, (inputPath) => {
+export function shouldSkipAgentTests(): boolean {
+	return process.env.AGENT_TESTS !== '1';
+}
+
+ssuite.optional(shouldSkipAgentTests, { title: 'toolCalling', location: 'panel' }, (inputPath) => {
 	const scenarioFolder = inputPath ?? path.join(__dirname, '..', 'test/scenarios/test-tools');
 	const scenarios = fetchToolScenarios(scenarioFolder);
 

test/simulation/baseline.json

@@ -939,6 +939,7 @@
   },
   {
     "name": "edit (toolCalling) [panel] - does not read",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 6,
     "failCount": 4,
@@ -1711,34 +1712,39 @@
   },
   {
     "name": "fetchWebPageTool (toolCalling) [panel] - multiple URLs boundary test with 6 URLs",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "fetchWebPageTool (toolCalling) [panel] - multiple URLs handling",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "fetchWebPageTool (toolCalling) [panel] - proper URL validation and query handling",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "fetchWebPageTool (toolCalling) [panel] - query parameter extraction",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "findFilesTool (toolCalling) [panel] - proper glob patterns",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 5,
     "failCount": 5,
@@ -9487,48 +9493,55 @@
   },
   {
     "name": "notebooks (toolCalling) [panel] - Edit cell tool",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "notebooks (toolCalling) [panel] - New Notebook Tool with EditFile and EditNotebook",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "notebooks (toolCalling) [panel] - New Notebook Tool without EditFile and with EditNotebook",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "notebooks (toolCalling) [panel] - New Notebook Tool without EditFile and without EditNotebook",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "notebooks (toolCalling) [panel] - Run cell at a specific index",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "notebooks (toolCalling) [panel] - Run cell tool",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "notebooks (toolCalling) [panel] - Run cell tool should avoid running markdown cells",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
@@ -11825,13 +11838,15 @@
   },
   {
     "name": "toolCalling [panel] - find all phone numbers in markdown files in the codebase",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 0,
     "failCount": 10,
     "score": 0
   },
   {
     "name": "toolCalling [panel] - I'm using git, create a new branch called issue-9876",
+    "optional": true,
     "contentFilterCount": 0,
     "passCount": 10,
     "failCount": 0,