Skip to content

Commit 19ec3d9

Browse files
salvatore-campagnasalvatore-campagna
authored
Return _ignored_source only if explicitly requested via stored_fields or fields (#114145)
We do not want to return `_ignored_source` with every search hit. Other than being expensive (especially when there are many ignored fields), it also exposes some implementation details. Anyway, it might still be useful to have the ability to retrieve it if necessary, at least for debugging purposes. For this reason we require that it is explicitly requested using `stored_fields` or `fields` and we do not return it by default or if requested via a wildcard `*` in `stored_fields` or `fields`.
1 parent 1708d9e commit 19ec3d9

File tree

2 files changed

+191
-1
lines changed

2 files changed

+191
-1
lines changed

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111

1212
import org.apache.lucene.index.LeafReaderContext;
1313
import org.elasticsearch.common.document.DocumentField;
14+
import org.elasticsearch.common.regex.Regex;
1415
import org.elasticsearch.index.mapper.IdFieldMapper;
1516
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
17+
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
1618
import org.elasticsearch.index.mapper.LegacyTypeFieldMapper;
1719
import org.elasticsearch.index.mapper.MappedFieldType;
1820
import org.elasticsearch.index.mapper.RoutingFieldMapper;
@@ -61,6 +63,21 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
6163
: fetchFieldsContext.fields().isEmpty() ? null
6264
: FieldFetcher.create(searchExecutionContext, fetchFieldsContext.fields());
6365

66+
// NOTE: Collect stored metadata fields requested via `fields` (in FetchFieldsContext`) like for instance the _ignored source field
67+
final Set<FieldAndFormat> fetchContextMetadataFields = new HashSet<>();
68+
if (fetchFieldsContext != null && fetchFieldsContext.fields() != null && fetchFieldsContext.fields().isEmpty() == false) {
69+
for (final FieldAndFormat fieldAndFormat : fetchFieldsContext.fields()) {
70+
// NOTE: _id and _source are always retrieved anyway, no need to do it explicitly. See FieldsVisitor.
71+
if (SourceFieldMapper.NAME.equals(fieldAndFormat.field) || IdFieldMapper.NAME.equals(fieldAndFormat.field)) {
72+
continue;
73+
}
74+
if (searchExecutionContext.isMetadataField(fieldAndFormat.field)
75+
&& searchExecutionContext.getFieldType(fieldAndFormat.field).isStored()) {
76+
fetchContextMetadataFields.add(fieldAndFormat);
77+
}
78+
}
79+
}
80+
6481
final FieldFetcher metadataFieldFetcher;
6582
if (storedFieldsContext != null
6683
&& storedFieldsContext.fieldNames() != null
@@ -73,6 +90,10 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
7390
continue;
7491
}
7592
final MappedFieldType fieldType = searchExecutionContext.getFieldType(matchingFieldName);
93+
// NOTE: Exclude _ignored_source when requested via wildcard '*'
94+
if (matchingFieldName.equals(IgnoredSourceFieldMapper.NAME) && Regex.isSimpleMatchPattern(storedField)) {
95+
continue;
96+
}
7697
// NOTE: checking if the field is stored is required for backward compatibility reasons and to make
7798
// sure we also handle here stored fields requested via `stored_fields`, which was previously a
7899
// responsibility of StoredFieldsPhase.
@@ -81,9 +102,14 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
81102
}
82103
}
83104
}
105+
// NOTE: Include also metadata stored fields requested via `fields`
106+
metadataFields.addAll(fetchContextMetadataFields);
84107
metadataFieldFetcher = FieldFetcher.create(searchExecutionContext, metadataFields);
85108
} else {
86-
metadataFieldFetcher = FieldFetcher.create(searchExecutionContext, DEFAULT_METADATA_FIELDS);
109+
// NOTE: Include also metadata stored fields requested via `fields`
110+
final Set<FieldAndFormat> allMetadataFields = new HashSet<>(DEFAULT_METADATA_FIELDS);
111+
allMetadataFields.addAll(fetchContextMetadataFields);
112+
metadataFieldFetcher = FieldFetcher.create(searchExecutionContext, allMetadataFields);
87113
}
88114
return new FetchSubPhaseProcessor() {
89115
@Override

‎x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/20_ignored_source.yml‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
---
2+
setup:
3+
- skip:
4+
features: headers
5+
- do:
6+
indices.create:
7+
index: test
8+
body:
9+
mappings:
10+
_source:
11+
mode: synthetic
12+
properties:
13+
object:
14+
enabled: false
15+
16+
- do:
17+
headers:
18+
Content-Type: application/yaml
19+
index:
20+
index: test
21+
id: "1"
22+
refresh: true
23+
body:
24+
object:
25+
name: "foo"
26+
value: 10
27+
28+
---
29+
"fetch stored fields wildcard":
30+
- do:
31+
headers:
32+
Content-Type: application/yaml
33+
search:
34+
index: test
35+
body:
36+
stored_fields: [ "*" ]
37+
38+
- match: { hits.total.value: 1 }
39+
- match: { hits.hits.0._ignored_source: null }
40+
41+
---
42+
"fetch fields wildcard":
43+
- do:
44+
headers:
45+
Content-Type: application/yaml
46+
search:
47+
index: test
48+
body:
49+
fields: [ "*" ]
50+
51+
- match: { hits.total.value: 1 }
52+
- match: { hits.hits.0._ignored_source: null }
53+
54+
---
55+
"fetch stored fields by name":
56+
- do:
57+
headers:
58+
Content-Type: application/yaml
59+
search:
60+
index: test
61+
body:
62+
stored_fields: [ _ignored_source ]
63+
64+
- match: { hits.total.value: 1 }
65+
- match: { hits.hits.0._ignored_source: !!binary "BgAAAG9iamVjdHktLS0KbmFtZTogImZvbyIKdmFsdWU6IDEwCg==" }
66+
67+
---
68+
"fetch fields by name":
69+
- do:
70+
headers:
71+
Content-Type: application/yaml
72+
search:
73+
index: test
74+
body:
75+
fields: [ _ignored_source ]
76+
77+
- match: { hits.total.value: 1 }
78+
- match: { hits.hits.0._ignored_source: !!binary "BgAAAG9iamVjdHktLS0KbmFtZTogImZvbyIKdmFsdWU6IDEwCg==" }
79+
80+
---
81+
"fields and stored fields combination":
82+
- do:
83+
headers:
84+
Content-Type: application/yaml
85+
search:
86+
index: test
87+
body:
88+
stored_fields: [ _ignored_source ]
89+
fields: [ _ignored_source ]
90+
query:
91+
match_all: {}
92+
93+
- match: { hits.total.value: 1 }
94+
- match: { hits.hits.0.fields.object: null }
95+
- match: { hits.hits.0._ignored_source: !!binary "BgAAAG9iamVjdHktLS0KbmFtZTogImZvbyIKdmFsdWU6IDEwCg==" }
96+
97+
---
98+
"wildcard fields and stored fields combination":
99+
- do:
100+
search:
101+
index: test
102+
body:
103+
stored_fields: [ "*" ]
104+
fields: [ "*" ]
105+
106+
- match: { hits.total.value: 1 }
107+
- match: { hits.hits.0._ignored_source: null }
108+
109+
---
110+
"fields with ignored source in stored fields":
111+
- do:
112+
headers:
113+
Content-Type: application/yaml
114+
search:
115+
index: test
116+
body:
117+
stored_fields: [ _ignored_source ]
118+
fields: [ object ]
119+
120+
- match: { hits.total.value: 1 }
121+
- match: { hits.hits.0._ignored_source: !!binary "BgAAAG9iamVjdHktLS0KbmFtZTogImZvbyIKdmFsdWU6IDEwCg==" }
122+
- match: { hits.hits.0.fields: null }
123+
124+
---
125+
"fields with ignored source in fields":
126+
- do:
127+
headers:
128+
Content-Type: application/yaml
129+
search:
130+
index: test
131+
body:
132+
stored_fields: [ object ]
133+
fields: [ _ignored_source ]
134+
135+
- match: { hits.total.value: 1 }
136+
- match: { hits.hits.0._ignored_source: !!binary "BgAAAG9iamVjdHktLS0KbmFtZTogImZvbyIKdmFsdWU6IDEwCg==" }
137+
138+
---
139+
"ignored source via fields and wildcard stored fields":
140+
- do:
141+
headers:
142+
Content-Type: application/yaml
143+
search:
144+
index: test
145+
body:
146+
stored_fields: [ "*" ]
147+
fields: [ _ignored_source ]
148+
149+
- match: { hits.total.value: 1 }
150+
- match: { hits.hits.0._ignored_source: !!binary "BgAAAG9iamVjdHktLS0KbmFtZTogImZvbyIKdmFsdWU6IDEwCg==" }
151+
152+
---
153+
"wildcard fields and ignored source via stored fields":
154+
- do:
155+
headers:
156+
Content-Type: application/yaml
157+
search:
158+
index: test
159+
body:
160+
stored_fields: [ _ignored_source ]
161+
fields: [ "*" ]
162+
163+
- match: { hits.total.value: 1 }
164+
- match: { hits.hits.0._ignored_source: !!binary "BgAAAG9iamVjdHktLS0KbmFtZTogImZvbyIKdmFsdWU6IDEwCg==" }

0 commit comments

Comments
 (0)