Skip to content

Commit d850a98

Browse files
romseygeekromseygeek
committed
flattened fields continue to not work here
1 parent 581a3db commit d850a98

File tree

9 files changed

+103
-32
lines changed

9 files changed

+103
-32
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,27 @@
88

99
package org.elasticsearch.index.mapper;
1010

11-
import org.elasticsearch.index.fielddata.ScriptDocValues;
1211
import org.elasticsearch.search.DocValueFormat;
1312
import org.elasticsearch.search.lookup.ValuesLookup;
1413

1514
import java.io.IOException;
16-
import java.util.ArrayList;
1715
import java.util.List;
1816

1917
/**
2018
* Value fetcher that loads from doc values.
2119
*/
2220
public final class DocValueFetcher implements ValueFetcher {
2321
private final DocValueFormat format;
24-
private final String field;
22+
private final MappedFieldType fieldType;
2523

26-
public DocValueFetcher(DocValueFormat format, String field) {
24+
public DocValueFetcher(DocValueFormat format, MappedFieldType fieldType) {
2725
this.format = format;
28-
this.field = field;
26+
this.fieldType = fieldType;
2927
}
3028

3129
@Override
3230
public List<Object> fetchValues(ValuesLookup lookup) throws IOException {
33-
ScriptDocValues<?> dv = lookup.doc().get(field, format);
34-
return new ArrayList<>(dv);
31+
return lookup.docValues(fieldType, format);
3532
}
3633

3734
}

‎server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public abstract class MappedFieldType {
5757

5858
private final String name;
5959
private final boolean docValues;
60-
private final boolean isIndexed;
60+
protected final boolean isIndexed;
6161
private final boolean isStored;
6262
private final TextSearchInfo textSearchInfo;
6363
private final Map<String, String> meta;
@@ -322,7 +322,7 @@ protected final void failIfNoDocValues() {
322322
}
323323
}
324324

325-
protected final void failIfNotIndexed() {
325+
protected void failIfNotIndexed() {
326326
if (isIndexed == false) {
327327
// we throw an IAE rather than an ISE so that it translates to a 4xx code rather than 5xx code on the http layer
328328
throw new IllegalArgumentException("Cannot search on field [" + name() + "] since it is not indexed.");

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) {
4747
}
4848
ValueFetcher fetcher = new DocValueFetcher(
4949
ft.docValueFormat(fieldAndFormat.format, null),
50-
ft.name()
50+
ft
5151
);
5252
fields.add(new DocValueField(fieldAndFormat.field, fetcher));
5353
}

‎server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,10 @@ public int hashCode() {
6969
}
7070
}
7171

72-
public ScriptDocValues<?> get(String field, DocValueFormat format) {
73-
FormatKey key = new FormatKey(field, format);
72+
public ScriptDocValues<?> get(MappedFieldType fieldType, DocValueFormat format) {
73+
FormatKey key = new FormatKey(fieldType.name(), format);
7474
ScriptDocValues<?> scriptValues = localCacheFormattedData.get(key);
7575
if (scriptValues == null) {
76-
final MappedFieldType fieldType = fieldTypeLookup.apply(field);
77-
if (fieldType == null) {
78-
throw new IllegalArgumentException("No field found for [" + field + "] in mapping");
79-
}
8076
// load fielddata on behalf of the script: otherwise it would need additional permissions
8177
// to deal with pagedbytes/ramusagestimator/etc
8278
scriptValues = AccessController.doPrivileged(new PrivilegedAction<ScriptDocValues<?>>() {

‎server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@
99
package org.elasticsearch.search.lookup;
1010

1111
import org.apache.lucene.index.LeafReaderContext;
12+
import org.elasticsearch.index.mapper.MappedFieldType;
13+
import org.elasticsearch.search.DocValueFormat;
1214

15+
import java.util.ArrayList;
16+
import java.util.List;
1317
import java.util.Map;
1418

1519
/**
@@ -51,6 +55,11 @@ public LeafDocLookup doc() {
5155
return this.docMap;
5256
}
5357

58+
@Override
59+
public List<Object> docValues(MappedFieldType fieldType, DocValueFormat format) {
60+
return new ArrayList<>(this.docMap.get(fieldType, format));
61+
}
62+
5463
public void setDocument(int docId) {
5564
docMap.setDocument(docId);
5665
sourceLookup.setSegmentAndDocument(ctx, docId);

‎server/src/main/java/org/elasticsearch/search/lookup/ValuesLookup.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@
1010

1111
import org.elasticsearch.common.bytes.BytesReference;
1212
import org.elasticsearch.common.xcontent.XContentType;
13+
import org.elasticsearch.index.mapper.MappedFieldType;
14+
import org.elasticsearch.search.DocValueFormat;
1315

16+
import java.util.List;
1417
import java.util.Map;
1518
import java.util.Objects;
1619

@@ -27,7 +30,7 @@ public interface ValuesLookup {
2730
/**
2831
* Returns a LeafDocLookup positioned on the current document
2932
*/
30-
LeafDocLookup doc();
33+
List<Object> docValues(MappedFieldType fieldType, DocValueFormat format);
3134

3235
/**
3336
* Override the source exposed by a ValuesLookup
@@ -43,8 +46,8 @@ public SourceLookup source() {
4346
}
4447

4548
@Override
46-
public LeafDocLookup doc() {
47-
return in.doc();
49+
public List<Object> docValues(MappedFieldType fieldType, DocValueFormat format) {
50+
return in.docValues(fieldType, format);
4851
}
4952
};
5053
}
@@ -62,7 +65,7 @@ public SourceLookup source() {
6265
}
6366

6467
@Override
65-
public LeafDocLookup doc() {
68+
public List<Object> docValues(MappedFieldType fieldType, DocValueFormat format) {
6669
throw new UnsupportedOperationException("FieldData is not available from a source-only lookup");
6770
}
6871
};
@@ -81,7 +84,7 @@ public SourceLookup source() {
8184
}
8285

8386
@Override
84-
public LeafDocLookup doc() {
87+
public List<Object> docValues(MappedFieldType fieldType, DocValueFormat format) {
8588
throw new UnsupportedOperationException("FieldData is not available from a source-only lookup");
8689
}
8790
};

‎test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@ protected final List<?> fetchFromDocValues(MapperService mapperService, MappedFi
305305
iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc());
306306
}, iw -> {
307307
SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup);
308-
ValueFetcher valueFetcher = new DocValueFetcher(format, ft.name());
308+
ValueFetcher valueFetcher = new DocValueFetcher(format, ft);
309309
IndexSearcher searcher = newSearcher(iw);
310310
LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
311311
LeafSearchLookup leaf = lookup.getLeafSearchLookup(context);

‎x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldSearchTests.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,7 @@ public void testLoadDocValuesFields() throws Exception {
411411
SearchResponse response = client().prepareSearch("test")
412412
.addDocValueField("flattened")
413413
.addDocValueField("flattened.key")
414+
.addDocValueField("flattened.other_key")
414415
.get();
415416
assertSearchResponse(response);
416417
assertHitCount(response, 1);
@@ -424,6 +425,10 @@ public void testLoadDocValuesFields() throws Exception {
424425
DocumentField keyedField = fields.get("flattened.key");
425426
assertEquals("flattened.key", keyedField.getName());
426427
assertEquals("value", keyedField.getValue());
428+
429+
DocumentField otherKeyedField = fields.get("flattened.other_key");
430+
assertEquals("flattened.other_key", otherKeyedField.getName());
431+
assertEquals("other_value", otherKeyedField.getValue());
427432
}
428433

429434
public void testFieldSort() throws Exception {

‎x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java‎

Lines changed: 71 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,16 @@
1111
import org.apache.lucene.index.LeafReaderContext;
1212
import org.apache.lucene.index.OrdinalMap;
1313
import org.apache.lucene.index.Term;
14+
import org.apache.lucene.search.AutomatonQuery;
1415
import org.apache.lucene.search.MultiTermQuery;
1516
import org.apache.lucene.search.PrefixQuery;
1617
import org.apache.lucene.search.Query;
1718
import org.apache.lucene.search.SortField;
19+
import org.apache.lucene.search.TermInSetQuery;
20+
import org.apache.lucene.search.TermQuery;
21+
import org.apache.lucene.search.TermRangeQuery;
1822
import org.apache.lucene.util.BytesRef;
23+
import org.elasticsearch.ElasticsearchException;
1924
import org.elasticsearch.common.lucene.Lucene;
2025
import org.elasticsearch.common.lucene.search.AutomatonQueries;
2126
import org.elasticsearch.common.unit.Fuzziness;
@@ -53,10 +58,13 @@
5358

5459
import java.io.IOException;
5560
import java.util.Arrays;
61+
import java.util.Collection;
5662
import java.util.List;
5763
import java.util.Map;
5864
import java.util.function.Supplier;
5965

66+
import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
67+
6068
/**
6169
* A field mapper that accepts a JSON object and flattens it into a single field. This data type
6270
* can be a useful alternative to an 'object' mapping when the object has a large, unknown set
@@ -162,17 +170,19 @@ public FlattenedFieldMapper build(ContentPath contentPath) {
162170
*/
163171
public static final class KeyedFlattenedFieldType extends StringFieldType {
164172
private final String key;
173+
private final String root;
165174

166-
public KeyedFlattenedFieldType(String name, boolean indexed, boolean hasDocValues, String key,
175+
public KeyedFlattenedFieldType(String root, boolean indexed, boolean hasDocValues, String key,
167176
boolean splitQueriesOnWhitespace, Map<String, String> meta) {
168-
super(name, indexed, false, hasDocValues,
177+
super(root + "." + key, indexed, false, hasDocValues,
169178
splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY,
170179
meta);
171180
this.key = key;
181+
this.root = root;
172182
}
173183

174-
private KeyedFlattenedFieldType(String name, String key, RootFlattenedFieldType ref) {
175-
this(name, ref.isSearchable(), ref.hasDocValues(), key, ref.splitQueriesOnWhitespace, ref.meta());
184+
private KeyedFlattenedFieldType(String root, String key, RootFlattenedFieldType ref) {
185+
this(root, ref.isSearchable(), ref.hasDocValues(), key, ref.splitQueriesOnWhitespace, ref.meta());
176186
}
177187

178188
@Override
@@ -186,7 +196,7 @@ public String key() {
186196

187197
@Override
188198
public Query existsQuery(SearchExecutionContext context) {
189-
Term term = new Term(name(), FlattenedFieldParser.createKeyedValue(key, ""));
199+
Term term = new Term(root, FlattenedFieldParser.createKeyedValue(key, ""));
190200
return new PrefixQuery(term);
191201
}
192202

@@ -205,8 +215,15 @@ public Query rangeQuery(Object lowerTerm,
205215
"] fields must include both an upper and a lower bound.");
206216
}
207217

208-
return super.rangeQuery(lowerTerm, upperTerm,
209-
includeLower, includeUpper, context);
218+
if (context.allowExpensiveQueries() == false) {
219+
throw new ElasticsearchException("[range] queries on [text] or [keyword] fields cannot be executed when '" +
220+
ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false.");
221+
}
222+
failIfNotIndexed();
223+
return new TermRangeQuery(root,
224+
indexedValueForSearch(lowerTerm),
225+
indexedValueForSearch(upperTerm),
226+
includeLower, includeUpper);
210227
}
211228

212229
@Override
@@ -234,7 +251,51 @@ public Query wildcardQuery(String value,
234251

235252
@Override
236253
public Query termQueryCaseInsensitive(Object value, SearchExecutionContext context) {
237-
return AutomatonQueries.caseInsensitiveTermQuery(new Term(name(), indexedValueForSearch(value)));
254+
return AutomatonQueries.caseInsensitiveTermQuery(new Term(root, indexedValueForSearch(value)));
255+
}
256+
257+
@Override
258+
public Query termQuery(Object value, SearchExecutionContext context) {
259+
failIfNotIndexed();
260+
return new TermQuery(new Term(root, indexedValueForSearch(value)));
261+
}
262+
263+
@Override
264+
public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
265+
failIfNotIndexed();
266+
BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new);
267+
return new TermInSetQuery(root, bytesRefs);
268+
}
269+
270+
@Override
271+
public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, SearchExecutionContext context) {
272+
if (context.allowExpensiveQueries() == false) {
273+
throw new ElasticsearchException("[prefix] queries cannot be executed when '" +
274+
ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false. For optimised prefix queries on text " +
275+
"fields please enable [index_prefixes].");
276+
}
277+
failIfNotIndexed();
278+
if (caseInsensitive) {
279+
AutomatonQuery query = AutomatonQueries.caseInsensitivePrefixQuery((new Term(root, indexedValueForSearch(value))));
280+
if (method != null) {
281+
query.setRewriteMethod(method);
282+
}
283+
return query;
284+
285+
}
286+
PrefixQuery query = new PrefixQuery(new Term(root, indexedValueForSearch(value)));
287+
if (method != null) {
288+
query.setRewriteMethod(method);
289+
}
290+
return query;
291+
}
292+
293+
@Override
294+
protected void failIfNotIndexed() {
295+
if (isIndexed == false) {
296+
// we throw an IAE rather than an ISE so that it translates to a 4xx code rather than 5xx code on the http layer
297+
throw new IllegalArgumentException("Cannot search on field [" + root + "] since it is not indexed.");
298+
}
238299
}
239300

240301
@Override
@@ -253,7 +314,7 @@ public BytesRef indexedValueForSearch(Object value) {
253314
@Override
254315
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
255316
failIfNoDocValues();
256-
return new KeyedFlattenedFieldData.Builder(name(), key, CoreValuesSourceType.KEYWORD);
317+
return new KeyedFlattenedFieldData.Builder(root + "._keyed", key, CoreValuesSourceType.KEYWORD);
257318
}
258319

259320
@Override
@@ -440,7 +501,7 @@ public RootFlattenedFieldType fieldType() {
440501

441502
@Override
442503
public KeyedFlattenedFieldType keyedFieldType(String key) {
443-
return new KeyedFlattenedFieldType(keyedFieldName(), key, fieldType());
504+
return new KeyedFlattenedFieldType(name(), key, fieldType());
444505
}
445506

446507
public String keyedFieldName() {

0 commit comments

Comments
 (0)