Skip to content

Commit 626b6e9

Browse files
lorenabalankibanamachine
lorenabalan
and
kibanamachine
authored
Improve FTR test coverage for Tools API (#232121)
## Summary * Closes elastic/search-team#10498 * Removes unused `testToolIds`. Tested locally via ``` yarn test:ftr --config x-pack/platform/test/onechat_api_integration/config.ts ``` and all tests passed. ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This was checked for breaking HTTP API changes, and any breaking changes have been approved by the breaking-change committee. The `release_note:breaking` label should be applied in these situations. - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) - [ ] Review the [backport guidelines](https://docs.google.com/document/d/1VyN5k91e5OVumlc0Gb9RPa3h1ewuPE705nRtioPiTvY/edit?usp=sharing) and apply applicable `backport:*` labels. ### Identify risks Does this PR introduce any risks? For example, consider risks like hard to test bugs, performance regression, potential of data loss. Describe the risk, its severity, and mitigation for each identified risk. Invite stakeholders and evaluate how to proceed before merging. - [ ] [See some risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) - [ ] ... --------- Co-authored-by: kibanamachine <[email protected]>
1 parent 1656c0a commit 626b6e9

File tree

5 files changed

+272
-3
lines changed

5 files changed

+272
-3
lines changed

‎x-pack/platform/test/onechat_api_integration/apis/builtin_tools.ts‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import expect from '@kbn/expect';
9+
import { builtinToolIds } from '@kbn/onechat-common';
10+
import type { FtrProviderContext } from '../../api_integration/ftr_provider_context';
11+
12+
export default function ({ getService }: FtrProviderContext) {
13+
const supertest = getService('supertest');
14+
const kibanaServer = getService('kibanaServer');
15+
16+
describe('Builtin Tools API', () => {
17+
describe('DELETE /api/chat/tools/.nl_search', () => {
18+
it('should return 400 error when attempting to delete any read-only builtin system tool', async () => {
19+
for (const toolId of Object.values(builtinToolIds) as string[]) {
20+
if (toolId === '.researcher_agent') {
21+
continue;
22+
}
23+
const response = await supertest
24+
.delete(`/api/chat/tools/${toolId}`)
25+
.set('kbn-xsrf', 'kibana')
26+
.expect(400);
27+
28+
expect(response.body).to.have.property('message');
29+
expect(response.body.message).to.eql(`Tool ${toolId} is read-only and can't be deleted`);
30+
}
31+
});
32+
33+
it('should return 404 when builtin tools API is disabled', async () => {
34+
await kibanaServer.uiSettings.update({
35+
'onechat:api:enabled': false,
36+
});
37+
38+
await supertest.delete('/api/chat/tools/.nl_search').set('kbn-xsrf', 'kibana').expect(404);
39+
40+
await kibanaServer.uiSettings.update({
41+
'onechat:api:enabled': true,
42+
});
43+
});
44+
});
45+
46+
describe('PUT /api/chat/tools/.nl_search', () => {
47+
it('should return 400 when attempting to update a builtin system tool', async () => {
48+
await supertest
49+
.put('/api/chat/tools/.nl_search')
50+
.set('kbn-xsrf', 'kibana')
51+
.send({ description: 'Updated description' })
52+
.expect(400);
53+
});
54+
});
55+
56+
describe('POST /api/chat/tools', () => {
57+
it('should return 400 when attempting to create a tool under an existing builtin tool ID', async () => {
58+
const toolData = {
59+
id: '.nl_search',
60+
type: 'esql',
61+
description: 'Attempting to create tool with builtin ID',
62+
configuration: {
63+
query: 'FROM test | LIMIT 10',
64+
params: {},
65+
},
66+
};
67+
68+
const response = await supertest
69+
.post('/api/chat/tools')
70+
.set('kbn-xsrf', 'kibana')
71+
.send(toolData)
72+
.expect(400);
73+
74+
expect(response.body).to.have.property('message');
75+
expect(response.body.message).to.contain('Tool id .nl_search is reserved');
76+
});
77+
});
78+
79+
describe('POST /api/chat/tools/_execute', () => {
80+
it('should execute a builtin tool successfully', async () => {
81+
const executeRequest = {
82+
tool_id: '.list_indices',
83+
tool_params: {},
84+
};
85+
86+
const response = await supertest
87+
.post('/api/chat/tools/_execute')
88+
.set('kbn-xsrf', 'kibana')
89+
.send(executeRequest)
90+
.expect(200);
91+
92+
expect(response.body).to.have.property('results');
93+
});
94+
95+
it('should return 404 when tools API is disabled', async () => {
96+
await kibanaServer.uiSettings.update({
97+
'onechat:api:enabled': false,
98+
});
99+
100+
const executeRequest = {
101+
tool_id: '.nl_search',
102+
tool_params: {
103+
query: 'test query',
104+
},
105+
};
106+
107+
await supertest
108+
.post('/api/chat/tools/_execute')
109+
.set('kbn-xsrf', 'kibana')
110+
.send(executeRequest)
111+
.expect(404);
112+
113+
await kibanaServer.uiSettings.update({
114+
'onechat:api:enabled': true,
115+
});
116+
});
117+
});
118+
});
119+
}

‎x-pack/platform/test/onechat_api_integration/apis/builtin_tools_internal.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import expect from '@kbn/expect';
9+
import { builtinToolIds } from '@kbn/onechat-common';
10+
import type { FtrProviderContext } from '../../api_integration/ftr_provider_context';
11+
12+
export default function ({ getService }: FtrProviderContext) {
13+
const supertest = getService('supertest');
14+
15+
describe('Builtin Tools internal API', () => {
16+
describe('POST /internal/chat/tools/_bulk_delete', () => {
17+
it('should return error results when attempting to bulk delete builtin system tools', async () => {
18+
const toolIds = Object.values(builtinToolIds).slice(0, 3) as string[];
19+
20+
const response = await supertest
21+
.post('/internal/chat/tools/_bulk_delete')
22+
.set('kbn-xsrf', 'kibana')
23+
.send({ ids: toolIds })
24+
.expect(200);
25+
26+
expect(response.body).to.have.property('results');
27+
expect(response.body.results).to.be.an('array');
28+
expect(response.body.results).to.have.length(3);
29+
30+
for (let i = 0; i < toolIds.length; i++) {
31+
const result = response.body.results[i];
32+
expect(result).to.have.property('toolId', toolIds[i]);
33+
expect(result).to.have.property('success', false);
34+
expect(result).to.have.property('reason');
35+
expect(result.reason).to.have.property('error');
36+
expect(result.reason.error).to.have.property('message');
37+
expect(result.reason.error.message).to.contain("is read-only and can't be deleted");
38+
}
39+
});
40+
41+
it('should handle mixed bulk delete with builtin and custom tools', async () => {
42+
const customTool = {
43+
id: 'test-custom-tool',
44+
type: 'esql',
45+
description: 'A test custom tool',
46+
tags: ['test'],
47+
configuration: {
48+
query: 'FROM test_index | LIMIT 10',
49+
params: {},
50+
},
51+
};
52+
53+
await supertest
54+
.post('/api/chat/tools')
55+
.set('kbn-xsrf', 'kibana')
56+
.send(customTool)
57+
.expect(200);
58+
59+
const mixedToolIds = ['.nl_search', 'test-custom-tool'];
60+
61+
const response = await supertest
62+
.post('/internal/chat/tools/_bulk_delete')
63+
.set('kbn-xsrf', 'kibana')
64+
.send({ ids: mixedToolIds })
65+
.expect(200);
66+
67+
expect(response.body).to.have.property('results');
68+
expect(response.body.results).to.be.an('array');
69+
expect(response.body.results).to.have.length(2);
70+
71+
// Builtin tool should fail
72+
const builtinResult = response.body.results.find((r: any) => r.toolId === '.nl_search');
73+
expect(builtinResult).to.have.property('success', false);
74+
expect(builtinResult.reason.error.message).to.contain("is read-only and can't be deleted");
75+
76+
// Custom tool should succeed
77+
const customResult = response.body.results.find(
78+
(r: any) => r.toolId === 'test-custom-tool'
79+
);
80+
expect(customResult).to.have.property('success', true);
81+
});
82+
});
83+
});
84+
}

‎x-pack/platform/test/onechat_api_integration/apis/esql_tools.ts‎

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ export default function ({ getService }: FtrProviderContext) {
1212
const supertest = getService('supertest');
1313
const kibanaServer = getService('kibanaServer');
1414
const log = getService('log');
15+
const es = getService('es');
1516

1617
describe('ES|QL Tools API', () => {
1718
const createdToolIds: string[] = [];
@@ -125,6 +126,71 @@ export default function ({ getService }: FtrProviderContext) {
125126
});
126127
});
127128

129+
describe('POST /api/chat/tools/_execute', () => {
130+
const testIndex = 'test-onechat-index';
131+
132+
before(async () => {
133+
await es.indices.create({
134+
index: testIndex,
135+
mappings: {
136+
properties: {
137+
name: { type: 'text' },
138+
age: { type: 'integer' },
139+
'@timestamp': { type: 'date' },
140+
},
141+
},
142+
});
143+
await es.bulk({
144+
body: [
145+
{ index: { _index: testIndex } },
146+
{ name: 'Test Case 1', age: 25, '@timestamp': '2023-01-01T00:00:00Z' },
147+
{ index: { _index: testIndex } },
148+
{ name: 'Test Case 2', age: 30, '@timestamp': '2023-01-02T00:00:00Z' },
149+
{ index: { _index: testIndex } },
150+
{ name: 'Test Case 3', age: 35, '@timestamp': '2023-01-03T00:00:00Z' },
151+
],
152+
});
153+
await es.indices.refresh({ index: testIndex });
154+
155+
const testTool = {
156+
type: 'esql',
157+
description: 'A test tool',
158+
tags: ['test'],
159+
configuration: {
160+
query: `FROM ${testIndex} | LIMIT 3`,
161+
params: {},
162+
},
163+
id: 'execute-test-tool',
164+
};
165+
166+
await supertest
167+
.post('/api/chat/tools')
168+
.set('kbn-xsrf', 'kibana')
169+
.send(testTool)
170+
.expect(200);
171+
172+
createdToolIds.push(testTool.id);
173+
});
174+
175+
it('should execute a new ES|QL tool successfully', async () => {
176+
const executeRequest = {
177+
tool_id: 'execute-test-tool',
178+
tool_params: {},
179+
};
180+
const response = await supertest
181+
.post('/api/chat/tools/_execute')
182+
.set('kbn-xsrf', 'kibana')
183+
.send(executeRequest)
184+
.expect(200);
185+
186+
expect(response.body).to.have.property('results');
187+
});
188+
189+
after(async () => {
190+
await es.indices.delete({ index: testIndex });
191+
});
192+
});
193+
128194
describe('GET /api/chat/tools/get-test-tool', () => {
129195
let testToolId: string;
130196

@@ -176,8 +242,6 @@ export default function ({ getService }: FtrProviderContext) {
176242
});
177243

178244
describe('GET /api/chat/tools', () => {
179-
const testToolIds: string[] = [];
180-
181245
before(async () => {
182246
for (let i = 0; i < 3; i++) {
183247
const testTool = {
@@ -191,7 +255,6 @@ export default function ({ getService }: FtrProviderContext) {
191255
.send(testTool)
192256
.expect(200);
193257

194-
testToolIds.push(testTool.id);
195258
createdToolIds.push(testTool.id);
196259
}
197260
});

‎x-pack/platform/test/onechat_api_integration/apis/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,7 @@ export default function ({ loadTestFile, getService }: FtrProviderContext) {
2626
loadTestFile(require.resolve('./esql_tools.ts'));
2727
loadTestFile(require.resolve('./esql_tools_internal.ts'));
2828
loadTestFile(require.resolve('./agents.ts'));
29+
loadTestFile(require.resolve('./builtin_tools.ts'));
30+
loadTestFile(require.resolve('./builtin_tools_internal.ts'));
2931
});
3032
}

‎x-pack/platform/test/tsconfig.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,5 +167,6 @@
167167
"@kbn/core-chrome-browser",
168168
"@kbn/licensing-types",
169169
"@kbn/product-intercept-plugin",
170+
"@kbn/onechat-common",
170171
]
171172
}

0 commit comments

Comments
 (0)