Skip to content

Support flexible field access pattern in ingest pipelines #133270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Aug 29, 2025
Merged

Support flexible field access pattern in ingest pipelines #133270

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
502088f
Add safe acquire method for access pattern
jbaiera Aug 13, 2025
0bde0ef
Expand ResolveResult with additional outcome
jbaiera Aug 13, 2025
f09c37c
Add new access pattern logic.
jbaiera Aug 20, 2025
601f970
Fix incorrect state usage inside ingest service
jbaiera Aug 20, 2025
c54262f
Add capability for flexible field access pattern in put ingest pipeline
jbaiera Aug 20, 2025
932c9a6
Add rest tests
jbaiera Aug 20, 2025
e631d75
[CI] Auto commit changes from spotless
Aug 20, 2025
ca70aec
Actually randomize the test
jbaiera Aug 22, 2025
9e0f209
Merge branch 'main' into streams-ingest-pipeline-field-access-pattern
jbaiera Aug 22, 2025
835a765
Remove unused resolve method
jbaiera Aug 25, 2025
2fb6c53
Plug uncovered test cases
jbaiera Aug 25, 2025
43b3c42
cleanup
jbaiera Aug 25, 2025
1299ad8
Merge branch 'main' into streams-ingest-pipeline-field-access-pattern
jbaiera Aug 26, 2025
7c5c853
Merge branch 'main' into streams-ingest-pipeline-field-access-pattern
jbaiera Aug 26, 2025
d408092
Merge branch 'main' into streams-ingest-pipeline-field-access-pattern
jbaiera Aug 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion modules/ingest-common/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ dependencies {

restResources {
restApi {
include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget', 'search', 'simulate'
include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget', 'search', 'simulate', 'capabilities'
}
}

Expand Down
381 changes: 381 additions & 0 deletions ...mmon/src/yamlRestTest/resources/rest-api-spec/test/ingest/340_flexible_access_pattern.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,381 @@
---
setup:
- requires:
reason: "Flexible access pattern was added in 9.2+"
test_runner_features: [ capabilities ]
capabilities:
- method: PUT
path: /_ingest/pipeline/{id}
capabilities: [ 'field_access_pattern.flexible' ]

---
teardown:
- do:
ingest.delete_pipeline:
id: "1"
ignore: 404

---
"Test dotted field name writes":
- do:
ingest.put_pipeline:
id: "1"
body: >
{
"field_access_pattern": "flexible",
"processors": [
{
"set": {
"field": "a.b.c.d",
"value": "1"
}
}
]
}
- match: { acknowledged: true }

- do:
index:
index: test
id: "no_field"
pipeline: "1"
body: {
foo: bar
}

- do:
index:
index: test
id: "normalized"
pipeline: "1"
body: {
a: {
b: {
c: {
d: 0
}
}
}
}

- do:
index:
index: test
id: "dotted_only"
pipeline: "1"
body: {
a.b.c.d: 0
}

- do:
index:
index: test
id: "split_dots"
pipeline: "1"
body: {
a.b: {
c.d: 0
}
}

- do:
index:
index: test
id: "middle_dot"
pipeline: "1"
body: {
a: {
b.c: {
d: 0
}
}
}

- do:
get:
index: test
id: "no_field"
- match: { _source.a\.b\.c\.d: "1" }
- do:
get:
index: test
id: "normalized"
- match: { _source.a.b.c.d: "1" }
- do:
get:
index: test
id: "dotted_only"
- match: { _source.a\.b\.c\.d: "1" }
- do:
get:
index: test
id: "split_dots"
- match: { _source.a\.b.c\.d: "1" }
- do:
get:
index: test
id: "middle_dot"
- match: { _source.a.b\.c.d: "1" }

---
"Test dotted field name retrieval":
- do:
ingest.put_pipeline:
id: "1"
body: >
{
"field_access_pattern": "flexible",
"processors": [
{
"set": {
"field": "result",
"copy_from": "a.b.c.d"
}
}
]
}
- match: { acknowledged: true }

- do:
index:
index: test
id: "normalized"
pipeline: "1"
body: {
a: {
b: {
c: {
d: 0
}
}
}
}

- do:
index:
index: test
id: "dotted_only"
pipeline: "1"
body: {
a.b.c.d: 0
}

- do:
index:
index: test
id: "split_dots"
pipeline: "1"
body: {
a.b: {
c.d: 0
}
}

- do:
index:
index: test
id: "middle_dot"
pipeline: "1"
body: {
a: {
b.c: {
d: 0
}
}
}

- do:
get:
index: test
id: "normalized"
- match: { _source.result: 0 }
- do:
get:
index: test
id: "dotted_only"
- match: { _source.result: 0 }
- do:
get:
index: test
id: "split_dots"
- match: { _source.result: 0 }
- do:
get:
index: test
id: "middle_dot"
- match: { _source.result: 0 }

---
"Test dotted field name exists":
- do:
ingest.put_pipeline:
id: "1"
body: >
{
"field_access_pattern": "flexible",
"processors": [
{
"rename": {
"field": "foo",
"target_field": "a.b.c.d",
"override": false
}
}
]
}
- match: { acknowledged: true }

- do:
catch: bad_request
index:
index: test
id: "normalized"
pipeline: "1"
body: {
foo: "bar",
a: {
b: {
c: {
d: 0
}
}
}
}
- match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" }

- do:
catch: bad_request
index:
index: test
id: "dotted_only"
pipeline: "1"
body: {
foo: "bar",
a.b.c.d: 0
}
- match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" }

- do:
catch: bad_request
index:
index: test
id: "split_dots"
pipeline: "1"
body: {
foo: "bar",
a.b: {
c.d: 0
}
}
- match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" }

- do:
catch: bad_request
index:
index: test
id: "middle_dot"
pipeline: "1"
body: {
foo: "bar",
a: {
b.c: {
d: 0
}
}
}
- match: { error.root_cause.0.reason: "field [a.b.c.d] already exists" }

---
"Test dotted field removal":
- do:
ingest.put_pipeline:
id: "1"
body: >
{
"field_access_pattern": "flexible",
"processors": [
{
"remove": {
"field": "a.b.c.d"
}
}
]
}
- match: { acknowledged: true }

- do:
index:
index: test
id: "normalized"
pipeline: "1"
body: {
foo: "bar",
a: {
b: {
c: {
d: 0
}
}
}
}

- do:
index:
index: test
id: "dotted_only"
pipeline: "1"
body: {
foo: "bar",
a.b.c.d: 0
}

- do:
index:
index: test
id: "split_dots"
pipeline: "1"
body: {
foo: "bar",
a.b: {
c.d: 0
}
}

- do:
index:
index: test
id: "middle_dot"
pipeline: "1"
body: {
foo: "bar",
a: {
b.c: {
d: 0
}
}
}

- do:
get:
index: test
id: "normalized"
- match: { _source.foo: "bar" }
- is_false: _source.a.b.c.d
- do:
get:
index: test
id: "dotted_only"
- match: { _source.foo: "bar" }
- is_false: _source.a\.b\.c\.d
- do:
get:
index: test
id: "split_dots"
- match: { _source.foo: "bar" }
- is_false: _source.a\.b.c\.d
- do:
get:
index: test
id: "middle_dot"
- match: { _source.foo: "bar" }
- is_false: _source.a.b\.c.d
Loading