Skip to content

Commit 04ea11b

Browse files
jbaieraelasticsearchmachine
jbaiera
and
elasticsearchmachine
authored
Support flexible field access pattern in ingest pipelines (elastic#133270)
Adds support for the flexible field access pattern within ingest pipelines, which provides the ability to progressively scan for fields on an ingest document in a way that includes dotted field names --------- Co-authored-by: elasticsearchmachine <[email protected]>
1 parent d3184ca commit 04ea11b

File tree

6 files changed

+2073
-635
lines changed

6 files changed

+2073
-635
lines changed

‎modules/ingest-common/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ dependencies {
2828

2929
restResources {
3030
restApi {
31-
include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget', 'search', 'simulate'
31+
include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget', 'search', 'simulate', 'capabilities'
3232
}
3333
}
3434

‎modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/340_flexible_access_pattern.yml‎

Lines changed: 381 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,381 @@
1+
---
2+
setup:
3+
- requires:
4+
reason: "Flexible access pattern was added in 9.2+"
5+
test_runner_features: [ capabilities ]
6+
capabilities:
7+
- method: PUT
8+
path: /_ingest/pipeline/{id}
9+
capabilities: [ 'field_access_pattern.flexible' ]
10+
11+
---
12+
teardown:
13+
- do:
14+
ingest.delete_pipeline:
15+
id: "1"
16+
ignore: 404
17+
18+
---
19+
"Test dotted field name writes":
20+
- do:
21+
ingest.put_pipeline:
22+
id: "1"
23+
body: >
24+
{
25+
"field_access_pattern": "flexible",
26+
"processors": [
27+
{
28+
"set": {
29+
"field": "a.b.c.d",
30+
"value": "1"
31+
}
32+
}
33+
]
34+
}
35+
- match: { acknowledged: true }
36+
37+
- do:
38+
index:
39+
index: test
40+
id: "no_field"
41+
pipeline: "1"
42+
body: {
43+
foo: bar
44+
}
45+
46+
- do:
47+
index:
48+
index: test
49+
id: "normalized"
50+
pipeline: "1"
51+
body: {
52+
a: {
53+
b: {
54+
c: {
55+
d: 0
56+
}
57+
}
58+
}
59+
}
60+
61+
- do:
62+
index:
63+
index: test
64+
id: "dotted_only"
65+
pipeline: "1"
66+
body: {
67+
a.b.c.d: 0
68+
}
69+
70+
- do:
71+
index:
72+
index: test
73+
id: "split_dots"
74+
pipeline: "1"
75+
body: {
76+
a.b: {
77+
c.d: 0
78+
}
79+
}
80+
81+
- do:
82+
index:
83+
index: test
84+
id: "middle_dot"
85+
pipeline: "1"
86+
body: {
87+
a: {
88+
b.c: {
89+
d: 0
90+
}
91+
}
92+
}
93+
94+
- do:
95+
get:
96+
index: test
97+
id: "no_field"
98+
- match: { _source.a\.b\.c\.d: "1" }
99+
- do:
100+
get:
101+
index: test
102+
id: "normalized"
103+
- match: { _source.a.b.c.d: "1" }
104+
- do:
105+
get:
106+
index: test
107+
id: "dotted_only"
108+
- match: { _source.a\.b\.c\.d: "1" }
109+
- do:
110+
get:
111+
index: test
112+
id: "split_dots"
113+
- match: { _source.a\.b.c\.d: "1" }
114+
- do:
115+
get:
116+
index: test
117+
id: "middle_dot"
118+
- match: { _source.a.b\.c.d: "1" }
119+
120+
---
121+
"Test dotted field name retrieval":
122+
- do:
123+
ingest.put_pipeline:
124+
id: "1"
125+
body: >
126+
{
127+
"field_access_pattern": "flexible",
128+
"processors": [
129+
{
130+
"set": {
131+
"field": "result",
132+
"copy_from": "a.b.c.d"
133+
}
134+
}
135+
]
136+
}
137+
- match: { acknowledged: true }
138+
139+
- do:
140+
index:
141+
index: test
142+
id: "normalized"
143+
pipeline: "1"
144+
body: {
145+
a: {
146+
b: {
147+
c: {
148+
d: 0
149+
}
150+
}
151+
}
152+
}
153+
154+
- do:
155+
index:
156+
index: test
157+
id: "dotted_only"
158+
pipeline: "1"
159+
body: {
160+
a.b.c.d: 0
161+
}
162+
163+
- do:
164+
index:
165+
index: test
166+
id: "split_dots"
167+
pipeline: "1"
168+
body: {
169+
a.b: {
170+
c.d: 0
171+
}
172+
}
173+
174+
- do:
175+
index:
176+
index: test
177+
id: "middle_dot"
178+
pipeline: "1"
179+
body: {
180+
a: {
181+
b.c: {
182+
d: 0
183+
}
184+
}
185+
}
186+
187+
- do:
188+
get:
189+
index: test
190+
id: "normalized"
191+
- match: { _source.result: 0 }
192+
- do:
193+
get:
194+
index: test
195+
id: "dotted_only"
196+
- match: { _source.result: 0 }
197+
- do:
198+
get:
199+
index: test
200+
id: "split_dots"
201+
- match: { _source.result: 0 }
202+
- do:
203+
get:
204+
index: test
205+
id: "middle_dot"
206+
- match: { _source.result: 0 }
207+
208+
---
209+
"Test dotted field name exists":
210+
- do:
211+
ingest.put_pipeline:
212+
id: "1"
213+
body: >
214+
{
215+
"field_access_pattern": "flexible",
216+
"processors": [
217+
{
218+
"rename": {
219+
"field": "foo",
220+
"target_field": "a.b.c.d",
221+
"override": false
222+
}
223+
}
224+
]
225+
}
226+
- match: { acknowledged: true }
227+
228+
- do:
229+
catch: bad_request
230+
index:
231+
index: test
232+
id: "normalized"
233+
pipeline: "1"
234+
body: {
235+
foo: "bar",
236+
a: {
237+
b: {
238+
c: {
239+
d: 0
240+
}
241+
}
242+
}
243+
}
244+
- match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" }
245+
246+
- do:
247+
catch: bad_request
248+
index:
249+
index: test
250+
id: "dotted_only"
251+
pipeline: "1"
252+
body: {
253+
foo: "bar",
254+
a.b.c.d: 0
255+
}
256+
- match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" }
257+
258+
- do:
259+
catch: bad_request
260+
index:
261+
index: test
262+
id: "split_dots"
263+
pipeline: "1"
264+
body: {
265+
foo: "bar",
266+
a.b: {
267+
c.d: 0
268+
}
269+
}
270+
- match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" }
271+
272+
- do:
273+
catch: bad_request
274+
index:
275+
index: test
276+
id: "middle_dot"
277+
pipeline: "1"
278+
body: {
279+
foo: "bar",
280+
a: {
281+
b.c: {
282+
d: 0
283+
}
284+
}
285+
}
286+
- match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" }
287+
288+
---
289+
"Test dotted field removal":
290+
- do:
291+
ingest.put_pipeline:
292+
id: "1"
293+
body: >
294+
{
295+
"field_access_pattern": "flexible",
296+
"processors": [
297+
{
298+
"remove": {
299+
"field": "a.b.c.d"
300+
}
301+
}
302+
]
303+
}
304+
- match: { acknowledged: true }
305+
306+
- do:
307+
index:
308+
index: test
309+
id: "normalized"
310+
pipeline: "1"
311+
body: {
312+
foo: "bar",
313+
a: {
314+
b: {
315+
c: {
316+
d: 0
317+
}
318+
}
319+
}
320+
}
321+
322+
- do:
323+
index:
324+
index: test
325+
id: "dotted_only"
326+
pipeline: "1"
327+
body: {
328+
foo: "bar",
329+
a.b.c.d: 0
330+
}
331+
332+
- do:
333+
index:
334+
index: test
335+
id: "split_dots"
336+
pipeline: "1"
337+
body: {
338+
foo: "bar",
339+
a.b: {
340+
c.d: 0
341+
}
342+
}
343+
344+
- do:
345+
index:
346+
index: test
347+
id: "middle_dot"
348+
pipeline: "1"
349+
body: {
350+
foo: "bar",
351+
a: {
352+
b.c: {
353+
d: 0
354+
}
355+
}
356+
}
357+
358+
- do:
359+
get:
360+
index: test
361+
id: "normalized"
362+
- match: { _source.foo: "bar" }
363+
- is_false: _source.a.b.c.d
364+
- do:
365+
get:
366+
index: test
367+
id: "dotted_only"
368+
- match: { _source.foo: "bar" }
369+
- is_false: _source.a\.b\.c\.d
370+
- do:
371+
get:
372+
index: test
373+
id: "split_dots"
374+
- match: { _source.foo: "bar" }
375+
- is_false: _source.a\.b.c\.d
376+
- do:
377+
get:
378+
index: test
379+
id: "middle_dot"
380+
- match: { _source.foo: "bar" }
381+
- is_false: _source.a.b\.c.d

0 commit comments

Comments
 (0)