Skip to content

Commit 74bf295

Browse files
twishabansaltwishabansal
authored
tests: add e2e tests for @toolbox/core (#13)
* feat: Add basic client and tool * cleanup * clean * add unit tests * increase test cov * test file cleanup * clean * small fix * Added unit tests for tool * cleanup * test cleanup * new tests * del e2e file * test * chore: lint (#11) * lint * lint * lint * move files to different PR * move files to different pr * move files to correct pr * lint * lint * lint * lint * lint * fix test file * lint * rebase * Update protocol.ts * package file cleanup * try * fix config * lint fix * cleanup * clean * fix tests * lint * lint * fix * fix config * fix * fix tests * lint * fix tests * fix docstrings * fix e2e tests * Update test.e2e.ts * Update client.ts * Update integration.cloudbuild.yaml * move files to correct pr * lint * lint * lint * lint * fix test file * lint * fix docstrings * Update client.ts * lint * fix any type errors * lint * clean * remove any * "any" type lint * change any to unknown * lint * remove unused deps * move deps to correct pr * rename methods * lint * fix method name * rename methods * resolve comments * lint * use parse instead of safeparse * use parse instead of safeparse * any issue fix * any lint * fix any type issue
1 parent cc6072b commit 74bf295

File tree

11 files changed

+1745
-40
lines changed

11 files changed

+1745
-40
lines changed

‎packages/toolbox-core/.eslintrc.json

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@
88
{
99
"files": [
1010
"**/test/**/*.ts",
11-
"**/*.test.ts"
11+
"**/*.test.ts",
12+
"**/jest.globalSetup.ts",
13+
"**/jest.globalTeardown.ts",
14+
"**/jest.setup.ts"
1215
],
1316
"rules": {
1417
"n/no-unpublished-import": "off"

‎packages/toolbox-core/integration.cloudbuild.yaml

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,19 @@ steps:
2727
args:
2828
- '-c'
2929
- npm run test:unit
30+
- id: Run integration tests
31+
name: 'node:${_VERSION}'
32+
entrypoint: /bin/bash
33+
dir: packages/toolbox-core
34+
env:
35+
- TOOLBOX_URL=$_TOOLBOX_URL
36+
- TOOLBOX_VERSION=$_TOOLBOX_VERSION
37+
- GOOGLE_CLOUD_PROJECT=$PROJECT_ID
38+
args:
39+
- '-c'
40+
- npm run test:e2e
3041
options:
3142
logging: CLOUD_LOGGING_ONLY
3243
substitutions:
3344
_VERSION: '20.0.0'
34-
_TOOLBOX_VERSION: '0.5.0'
45+
_TOOLBOX_VERSION: '0.5.0'

‎packages/toolbox-core/jest.e2e.config.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"preset": "ts-jest",
3+
"testEnvironment": "node",
4+
"testMatch": [
5+
"<rootDir>/test/e2e/*.e2e.ts"
6+
],
7+
"globalSetup": "<rootDir>/test/e2e/jest.globalSetup.ts",
8+
"globalTeardown": "<rootDir>/test/e2e/jest.globalTeardown.ts",
9+
"testTimeout": 60000,
10+
"collectCoverage": false
11+
}

‎packages/toolbox-core/package-lock.json

Lines changed: 1277 additions & 36 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/toolbox-core/package.json

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,22 @@
3434
"compile": "tsc -p .",
3535
"prepare": "npm run compile",
3636
"test:unit": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --config jest.config.json",
37+
"test:e2e": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --config jest.e2e.config.json --runInBand",
3738
"coverage": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --config jest.config.json --coverage"
3839
},
3940
"devDependencies": {
41+
"@google-cloud/secret-manager": "^6.0.1",
42+
"@google-cloud/storage": "^7.16.0",
43+
"@types/fs-extra": "^11.0.4",
4044
"@types/jest": "^29.5.14",
45+
"@types/tmp": "^0.2.6",
4146
"cross-env": "^7.0.3",
47+
"fs-extra": "^11.3.0",
48+
"google-auth-library": "^9.15.1",
4249
"gts": "^5.3.1",
4350
"jest": "^29.7.0",
44-
"ts-jest": "^29.3.2",
51+
"tmp": "^0.2.3",
52+
"ts-jest": "^29.3.4",
4553
"typescript": "^5.8.3"
4654
},
4755
"dependencies": {

‎packages/toolbox-core/test/e2e/jest.globalSetup.ts

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import * as path from 'path';
16+
import * as fs from 'fs-extra';
17+
import {spawn} from 'child_process';
18+
import {
19+
getEnvVar,
20+
accessSecretVersion,
21+
createTmpFile,
22+
downloadBlob,
23+
getToolboxBinaryGcsPath,
24+
delay,
25+
} from './utils';
26+
import {CustomGlobal} from './types';
27+
28+
const TOOLBOX_BINARY_NAME = 'toolbox';
29+
const SERVER_READY_TIMEOUT_MS = 30000; // 30 seconds
30+
const SERVER_READY_POLL_INTERVAL_MS = 2000; // 2 seconds
31+
32+
export default async function globalSetup(): Promise<void> {
33+
console.log('\nJest Global Setup: Starting...');
34+
35+
try {
36+
const projectId = getEnvVar('GOOGLE_CLOUD_PROJECT');
37+
const toolboxVersion = getEnvVar('TOOLBOX_VERSION');
38+
39+
// Fetch tools manifest and create temp file
40+
const toolsManifest = await accessSecretVersion(
41+
projectId,
42+
'sdk_testing_tools'
43+
);
44+
const toolsFilePath = await createTmpFile(toolsManifest);
45+
(globalThis as CustomGlobal).__TOOLS_FILE_PATH__ = toolsFilePath;
46+
console.log(`Tools manifest stored at: ${toolsFilePath}`);
47+
48+
// Download toolbox binary
49+
const toolboxGcsPath = getToolboxBinaryGcsPath(toolboxVersion);
50+
const localToolboxPath = path.resolve(__dirname, TOOLBOX_BINARY_NAME);
51+
52+
console.log(
53+
`Downloading toolbox binary from gs://genai-toolbox/${toolboxGcsPath} to ${localToolboxPath}...`
54+
);
55+
await downloadBlob('genai-toolbox', toolboxGcsPath, localToolboxPath);
56+
console.log('Toolbox binary downloaded successfully.');
57+
58+
// Make toolbox executable
59+
await fs.chmod(localToolboxPath, 0o700);
60+
61+
// Start toolbox server
62+
console.log('Starting toolbox server process...');
63+
const serverProcess = spawn(
64+
localToolboxPath,
65+
['--tools_file', toolsFilePath],
66+
{
67+
stdio: ['ignore', 'pipe', 'pipe'], // ignore stdin, pipe stdout/stderr
68+
}
69+
);
70+
71+
(globalThis as CustomGlobal).__TOOLBOX_SERVER_PROCESS__ = serverProcess;
72+
73+
serverProcess.stdout?.on('data', (data: Buffer) => {
74+
console.log(`[ToolboxServer STDOUT]: ${data.toString().trim()}`);
75+
});
76+
77+
serverProcess.stderr?.on('data', (data: Buffer) => {
78+
console.error(`[ToolboxServer STDERR]: ${data.toString().trim()}`);
79+
});
80+
81+
serverProcess.on('error', err => {
82+
console.error('Toolbox server process error:', err);
83+
throw new Error('Failed to start toolbox server process.');
84+
});
85+
86+
serverProcess.on('exit', (code, signal) => {
87+
console.log(
88+
`Toolbox server process exited with code ${code}, signal ${signal}.`
89+
);
90+
if (
91+
(globalThis as CustomGlobal).__TOOLBOX_SERVER_PROCESS__ &&
92+
!(globalThis as CustomGlobal).__SERVER_TEARDOWN_INITIATED__
93+
) {
94+
console.error('Toolbox server exited prematurely during setup.');
95+
}
96+
});
97+
98+
// Wait for server to start (basic poll check)
99+
let started = false;
100+
const startTime = Date.now();
101+
while (Date.now() - startTime < SERVER_READY_TIMEOUT_MS) {
102+
if (
103+
serverProcess.pid &&
104+
!serverProcess.killed &&
105+
serverProcess.exitCode === null
106+
) {
107+
console.log(
108+
'Toolbox server process appears to be running. Polling for stability...'
109+
);
110+
await delay(SERVER_READY_POLL_INTERVAL_MS * 2);
111+
if (serverProcess.exitCode === null) {
112+
console.log(
113+
'Toolbox server started successfully (process is active).'
114+
);
115+
started = true;
116+
break;
117+
} else {
118+
console.log('Toolbox server process exited after initial start.');
119+
break;
120+
}
121+
}
122+
await delay(SERVER_READY_POLL_INTERVAL_MS);
123+
console.log('Checking if toolbox server is started...');
124+
}
125+
126+
if (!started) {
127+
if (serverProcess && !serverProcess.killed) {
128+
serverProcess.kill('SIGTERM');
129+
}
130+
throw new Error(
131+
`Toolbox server failed to start within ${SERVER_READY_TIMEOUT_MS / 1000} seconds.`
132+
);
133+
}
134+
135+
console.log('Jest Global Setup: Completed successfully.');
136+
} catch (error) {
137+
console.error('Jest Global Setup Failed:', error);
138+
// Attempt to kill server if it started partially
139+
const serverProcess = (globalThis as CustomGlobal)
140+
.__TOOLBOX_SERVER_PROCESS__;
141+
if (serverProcess && !serverProcess.killed) {
142+
console.log('Attempting to terminate partially started server...');
143+
serverProcess.kill('SIGKILL');
144+
}
145+
// Clean up temp file if created
146+
const toolsFilePath = (globalThis as CustomGlobal).__TOOLS_FILE_PATH__;
147+
if (toolsFilePath) {
148+
try {
149+
await fs.remove(toolsFilePath);
150+
} catch (e) {
151+
console.error(
152+
'Error removing temp tools file during setup failure:',
153+
e
154+
);
155+
}
156+
}
157+
throw error;
158+
}
159+
}

‎packages/toolbox-core/test/e2e/jest.globalTeardown.ts

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import * as fs from 'fs-extra';
16+
import {CustomGlobal} from './types';
17+
18+
const SERVER_TERMINATE_TIMEOUT_MS = 10000; // 10 seconds
19+
20+
export default async function globalTeardown(): Promise<void> {
21+
console.log('\nJest Global Teardown: Starting...');
22+
(globalThis as CustomGlobal).__SERVER_TEARDOWN_INITIATED__ = true;
23+
24+
const customGlobal = globalThis as CustomGlobal;
25+
const serverProcess = customGlobal.__TOOLBOX_SERVER_PROCESS__;
26+
const toolsFilePath = customGlobal.__TOOLS_FILE_PATH__;
27+
28+
if (serverProcess && !serverProcess.killed) {
29+
console.log('Stopping toolbox server process...');
30+
serverProcess.kill('SIGTERM'); // Graceful termination
31+
32+
// Wait for the process to exit
33+
const stopPromise = new Promise<void>((resolve, reject) => {
34+
const timeout = setTimeout(() => {
35+
if (!serverProcess.killed) {
36+
console.warn(
37+
'Toolbox server did not terminate gracefully, sending SIGKILL.'
38+
);
39+
serverProcess.kill('SIGKILL');
40+
}
41+
// Resolve even if SIGKILL is needed, as we want teardown to finish
42+
resolve();
43+
}, SERVER_TERMINATE_TIMEOUT_MS);
44+
45+
serverProcess.on('exit', (code, signal) => {
46+
clearTimeout(timeout);
47+
console.log(
48+
`Toolbox server process exited with code ${code}, signal ${signal} during teardown.`
49+
);
50+
resolve();
51+
});
52+
serverProcess.on('error', err => {
53+
// Should not happen if already running
54+
clearTimeout(timeout);
55+
console.error('Error during server process termination:', err);
56+
reject(err);
57+
});
58+
});
59+
60+
try {
61+
await stopPromise;
62+
} catch (error) {
63+
console.error('Error while waiting for server to stop:', error);
64+
if (!serverProcess.killed) serverProcess.kill('SIGKILL'); // Ensure it's killed
65+
}
66+
} else {
67+
console.log('Toolbox server process was not running or already handled.');
68+
}
69+
70+
if (toolsFilePath) {
71+
try {
72+
console.log(`Removing temporary tools file: ${toolsFilePath}`);
73+
await fs.remove(toolsFilePath);
74+
} catch (error) {
75+
console.error(
76+
`Failed to remove temporary tools file ${toolsFilePath}:`,
77+
error
78+
);
79+
}
80+
}
81+
customGlobal.__TOOLBOX_SERVER_PROCESS__ = undefined;
82+
customGlobal.__TOOLS_FILE_PATH__ = undefined;
83+
84+
console.log('Jest Global Teardown: Completed.');
85+
}

‎packages/toolbox-core/test/e2e/test.e2e.ts

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import {ToolboxClient} from '../../src/toolbox_core/client';
16+
import {ToolboxTool} from '../../src/toolbox_core/tool';
17+
18+
describe('ToolboxClient E2E Tests', () => {
19+
let commonToolboxClient: ToolboxClient;
20+
let getNRowsTool: ReturnType<typeof ToolboxTool>;
21+
22+
beforeAll(async () => {
23+
commonToolboxClient = new ToolboxClient('http://localhost:5000');
24+
});
25+
26+
beforeEach(async () => {
27+
getNRowsTool = await commonToolboxClient.loadTool('get-n-rows');
28+
expect(getNRowsTool.getName()).toBe('get-n-rows');
29+
});
30+
31+
describe('invokeTool', () => {
32+
it('should invoke the getNRowsTool', async () => {
33+
const response = await getNRowsTool({num_rows: '2'});
34+
const result = response['result'];
35+
expect(typeof result).toBe('string');
36+
expect(result).toContain('row1');
37+
expect(result).toContain('row2');
38+
expect(result).not.toContain('row3');
39+
});
40+
41+
it('should invoke the getNRowsTool with missing params', async () => {
42+
await expect(getNRowsTool()).rejects.toThrow(
43+
/Argument validation failed for tool "get-n-rows":\s*- num_rows: Required/
44+
);
45+
});
46+
47+
it('should invoke the getNRowsTool with wrong param type', async () => {
48+
await expect(getNRowsTool({num_rows: 2})).rejects.toThrow(
49+
/Argument validation failed for tool "get-n-rows":\s*- num_rows: Expected string, received number/
50+
);
51+
});
52+
});
53+
});

‎packages/toolbox-core/test/e2e/types.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import {ChildProcess} from 'child_process';
16+
17+
// Used in jest global setup and teardown
18+
export type CustomGlobal = typeof globalThis & {
19+
__TOOLS_FILE_PATH__?: string;
20+
__TOOLBOX_SERVER_PROCESS__?: ChildProcess;
21+
__SERVER_TEARDOWN_INITIATED__?: boolean;
22+
};

0 commit comments

Comments
 (0)