Skip to content

Commit b435313

Browse files
sddonnesddonne
authored
[ES|QL] Add AST support for new FUSE parameters (elastic#237485)
Part of elastic#236938 ## Summary This PR adds `AST` support for the new`FUSE` parameters. ``` fuseCommand : DEV_FUSE (fuseType=identifier)? (fuseConfiguration)* ; fuseConfiguration : SCORE BY score=qualifiedName | KEY BY key=fields | GROUP BY group=qualifiedName | WITH options=mapExpression ; ``` `FUSE <fuse_method> SCORE BY <score_column> GROUP BY <group_column> KEY BY <key_columns> WITH <options>` ### Some Examples `| FUSE LINEAR # Merge results using linear combination of scores (equal weights by default)` `| FUSE LINEAR WITH { "normalizer": "minmax" } # Linear combination with min-max normalization (scales scores to 0-1 range) ` `| FUSE LINEAR WITH { "weights": { "fork1": 0.7, "fork2": 0.3 }, "normalizer": "minmax" } # Weighted linear combination: 70% lexical, 30% semantic, with min-max normalization` ### Note No tests has been added to check parameters or fields with dots, as the parser needs this [fix](elastic/elasticsearch#135901) to be merged to support them. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
1 parent 80f632f commit b435313

File tree

3 files changed

+427
-20
lines changed

3 files changed

+427
-20
lines changed

‎src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/fuse.test.ts‎

Lines changed: 307 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,24 @@ import { parse } from '../parser';
1212
describe('FUSE', () => {
1313
describe('correctly formatted', () => {
1414
it('can parse FUSE command without modifiers', () => {
15+
const text = `FROM index | FUSE`;
16+
17+
const { root, errors } = parse(text);
18+
19+
expect(errors.length).toBe(0);
20+
expect(root.commands[1]).toMatchObject({
21+
type: 'command',
22+
name: 'fuse',
23+
args: [],
24+
});
25+
});
26+
27+
it('can parse FUSE with a type', () => {
1528
const text = `FROM search-movies METADATA _score, _id, _index
1629
| FORK
1730
( WHERE semantic_title:"Shakespeare" | SORT _score)
1831
( WHERE title:"Shakespeare" | SORT _score)
19-
| FUSE
32+
| FUSE rrf
2033
| KEEP title, _score`;
2134

2235
const { root, errors } = parse(text);
@@ -25,18 +38,165 @@ describe('FUSE', () => {
2538
expect(root.commands[2]).toMatchObject({
2639
type: 'command',
2740
name: 'fuse',
28-
args: [],
41+
args: [
42+
{
43+
type: 'identifier',
44+
name: 'rrf',
45+
},
46+
],
47+
});
48+
});
49+
50+
it('can parse FUSE with SCORE BY', () => {
51+
const text = `FROM index | FUSE SCORE BY new_score`;
52+
53+
const { root, errors } = parse(text);
54+
55+
expect(errors.length).toBe(0);
56+
expect(root.commands[1]).toMatchObject({
57+
type: 'command',
58+
name: 'fuse',
59+
args: [
60+
{
61+
type: 'option',
62+
name: 'score by',
63+
args: [{ type: 'column', name: 'new_score' }],
64+
incomplete: false,
65+
},
66+
],
67+
incomplete: false,
68+
});
69+
});
70+
71+
it('can parse FUSE with KEY BY', () => {
72+
const text = `FROM index | FUSE KEY BY field1, field2`;
73+
74+
const { root, errors } = parse(text);
75+
76+
expect(errors.length).toBe(0);
77+
expect(root.commands[1]).toMatchObject({
78+
type: 'command',
79+
name: 'fuse',
80+
args: [
81+
{
82+
type: 'option',
83+
name: 'key by',
84+
args: [
85+
{ type: 'column', name: 'field1' },
86+
{ type: 'column', name: 'field2' },
87+
],
88+
incomplete: false,
89+
},
90+
],
91+
incomplete: false,
92+
});
93+
});
94+
95+
it('can parse FUSE with GROUP BY', () => {
96+
const text = `FROM index | FUSE GROUP BY group_field`;
97+
98+
const { root, errors } = parse(text);
99+
100+
expect(errors.length).toBe(0);
101+
expect(root.commands[1]).toMatchObject({
102+
type: 'command',
103+
name: 'fuse',
104+
args: [
105+
{
106+
type: 'option',
107+
name: 'group by',
108+
args: [{ type: 'column', name: 'group_field' }],
109+
incomplete: false,
110+
},
111+
],
112+
incomplete: false,
113+
});
114+
});
115+
116+
it('can parse FUSE with WITH', () => {
117+
const text = `FROM index | FUSE WITH { "normalizer": "minmax" }`;
118+
119+
const { root, errors } = parse(text);
120+
121+
expect(errors.length).toBe(0);
122+
expect(root.commands[1]).toMatchObject({
123+
type: 'command',
124+
name: 'fuse',
125+
args: [
126+
{
127+
type: 'option',
128+
name: 'with',
129+
args: [
130+
{
131+
type: 'map',
132+
entries: [
133+
{
134+
type: 'map-entry',
135+
key: { valueUnquoted: 'normalizer' },
136+
value: { type: 'literal', literalType: 'keyword', value: '"minmax"' },
137+
},
138+
],
139+
},
140+
],
141+
},
142+
],
143+
});
144+
});
145+
146+
it('can parse FUSE with all modifiers', () => {
147+
const text = `FROM index
148+
| FUSE rrf SCORE BY new_score KEY BY k1, k2 GROUP BY g WITH { "normalizer": "minmax" }`;
149+
150+
const { root, errors } = parse(text);
151+
152+
expect(errors.length).toBe(0);
153+
expect(root.commands[1]).toMatchObject({
154+
type: 'command',
155+
name: 'fuse',
156+
args: [
157+
{ type: 'identifier', name: 'rrf' },
158+
{
159+
type: 'option',
160+
name: 'score by',
161+
args: [{ type: 'column', name: 'new_score' }],
162+
},
163+
{
164+
type: 'option',
165+
name: 'key by',
166+
args: [
167+
{ type: 'column', name: 'k1' },
168+
{ type: 'column', name: 'k2' },
169+
],
170+
},
171+
{
172+
type: 'option',
173+
name: 'group by',
174+
args: [{ type: 'column', name: 'g' }],
175+
},
176+
{
177+
type: 'option',
178+
name: 'with',
179+
args: [
180+
{
181+
type: 'map',
182+
entries: [
183+
{
184+
type: 'map-entry',
185+
key: { valueUnquoted: 'normalizer' },
186+
value: { type: 'literal', literalType: 'keyword', value: '"minmax"' },
187+
},
188+
],
189+
},
190+
],
191+
},
192+
],
29193
});
30194
});
31195
});
32196

33197
describe('when incorrectly formatted, return errors', () => {
34198
it('when no pipe after', () => {
35-
const text = `FROM search-movies METADATA _score, _id, _index
36-
| FORK
37-
( WHERE semantic_title:"Shakespeare" | SORT _score)
38-
( WHERE title:"Shakespeare" | SORT _score)
39-
| FUSE KEEP title, _score`;
199+
const text = `FROM index | FUSE KEEP title, _score`;
40200

41201
const { errors } = parse(text);
42202

@@ -53,5 +213,145 @@ describe('FUSE', () => {
53213

54214
expect(errors.length > 0).toBe(true);
55215
});
216+
217+
it('can parse FUSE with incomplete SCORE BY', () => {
218+
const text = `FROM index | FUSE SCORE BY `;
219+
220+
const { root, errors } = parse(text);
221+
222+
expect(errors.length).toBe(1);
223+
expect(root.commands[1]).toMatchObject({
224+
type: 'command',
225+
name: 'fuse',
226+
args: [
227+
{
228+
type: 'option',
229+
name: 'score by',
230+
args: [],
231+
incomplete: true,
232+
},
233+
],
234+
incomplete: true,
235+
});
236+
});
237+
238+
it('can parse FUSE with incomplete KEY BY ', () => {
239+
const text = `FROM index | FUSE KEY BY `;
240+
241+
const { root, errors } = parse(text);
242+
243+
expect(errors.length).toBe(1);
244+
expect(root.commands[1]).toMatchObject({
245+
type: 'command',
246+
name: 'fuse',
247+
args: [
248+
{
249+
type: 'option',
250+
name: 'key by',
251+
args: [],
252+
incomplete: true,
253+
},
254+
],
255+
incomplete: true,
256+
});
257+
});
258+
259+
it('can parse FUSE with incomplete GROUP BY', () => {
260+
const text = `FROM index | FUSE GROUP BY `;
261+
262+
const { root, errors } = parse(text);
263+
264+
expect(errors.length).toBe(1);
265+
expect(root.commands[1]).toMatchObject({
266+
type: 'command',
267+
name: 'fuse',
268+
args: [
269+
{
270+
type: 'option',
271+
name: 'group by',
272+
args: [],
273+
incomplete: true,
274+
},
275+
],
276+
incomplete: true,
277+
});
278+
});
279+
280+
it('can parse FUSE with incomplete WITH', () => {
281+
const text = `FROM index | FUSE WITH `;
282+
283+
const { root, errors } = parse(text);
284+
285+
expect(errors.length).toBe(1);
286+
expect(root.commands[1]).toMatchObject({
287+
type: 'command',
288+
name: 'fuse',
289+
args: [
290+
{
291+
type: 'option',
292+
name: 'with',
293+
args: [],
294+
incomplete: true,
295+
},
296+
],
297+
incomplete: true,
298+
});
299+
});
300+
301+
it('can parse FUSE with incomplete WITH map expression', () => {
302+
const text = `FROM index | FUSE WITH {"normalizer":}`;
303+
304+
const { root, errors } = parse(text);
305+
306+
expect(errors.length).toBe(1);
307+
expect(root.commands[1]).toMatchObject({
308+
type: 'command',
309+
name: 'fuse',
310+
args: [
311+
{
312+
type: 'option',
313+
name: 'with',
314+
args: [
315+
{
316+
type: 'map',
317+
incomplete: true,
318+
},
319+
],
320+
incomplete: true,
321+
},
322+
],
323+
incomplete: true,
324+
});
325+
});
326+
327+
// This one is a syntactic valid query, but it's semantically invalid
328+
// The parser should not be responsible for catching this kind of error
329+
// However, we still want to make sure the AST is correctly generated
330+
it('can parse FUSE with duplicated modifiers', () => {
331+
const text = `FROM index | FUSE SCORE BY s1 SCORE BY s2`;
332+
333+
const { root, errors } = parse(text);
334+
335+
expect(errors.length).toBe(0);
336+
expect(root.commands[1]).toMatchObject({
337+
type: 'command',
338+
name: 'fuse',
339+
args: [
340+
{
341+
type: 'option',
342+
name: 'score by',
343+
args: [{ type: 'column', name: 's1' }],
344+
incomplete: false,
345+
},
346+
{
347+
type: 'option',
348+
name: 'score by',
349+
args: [{ type: 'column', name: 's2' }],
350+
incomplete: false,
351+
},
352+
],
353+
incomplete: false,
354+
});
355+
});
56356
});
57357
});

0 commit comments

Comments
 (0)