Fixes to fielddata and valuefetcher

Author: parkertimmins

test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java

@@ -8,21 +8,14 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesSkipIndexType;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.FieldLookup;
@@ -39,7 +32,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
-import java.util.function.Supplier;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -79,24 +71,6 @@ public static List<?> fetchSourceValue(MappedFieldType fieldType, Object sourceV
         return fetcher.fetchValues(source, -1, new ArrayList<>());
     }
 
-    public static List<?> fetchDocValues(MappedFieldType fieldType, Supplier<Document> documentSupplier) throws IOException {
-        IndexWriterConfig iwc = new IndexWriterConfig(null);
-        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc)) {
-            iw.addDocument(documentSupplier.get());
-            try (DirectoryReader reader = iw.getReader()) {
-                IndexSearcher searcher = newSearcher(reader);
-                LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
-                SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
-                when(searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH)).thenReturn(
-                    fieldType.fielddataBuilder(null).build(null, null)
-                );
-                ValueFetcher valueFetcher = fieldType.valueFetcher(searchExecutionContext, null);
-                valueFetcher.setNextReader(context);
-                return valueFetcher.fetchValues(null, 0, new ArrayList<>());
-            }
-        }
-    }
-
     public static List<?> fetchSourceValues(MappedFieldType fieldType, Object... values) throws IOException {
         String field = fieldType.name();
         SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);

x-pack/plugin/mapper-patterned-text/src/main/java/org/elasticsearch/xpack/patternedtext/PatternedTextFieldMapper.java

@@ -104,7 +104,7 @@ private PatternedTextFieldMapper(
     ) {
         super(simpleName, mappedFieldPatternedTextFieldType, builderParams);
         assert mappedFieldPatternedTextFieldType.getTextSearchInfo().isTokenized();
-        assert mappedFieldPatternedTextFieldType.hasDocValues();
+        assert mappedFieldPatternedTextFieldType.hasDocValues() == false;
         this.fieldType = Defaults.FIELD_TYPE;
         this.indexCreatedVersion = builder.indexCreatedVersion;
         this.indexAnalyzers = builder.analyzers.indexAnalyzers;

x-pack/plugin/mapper-patterned-text/src/main/java/org/elasticsearch/xpack/patternedtext/PatternedTextFieldType.java

@@ -27,13 +27,12 @@
 import org.elasticsearch.common.CheckedIntFunction;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
+import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData;
 import org.elasticsearch.index.mapper.BlockDocValuesReader;
 import org.elasticsearch.index.mapper.BlockLoader;
-import org.elasticsearch.index.mapper.DocValueFetcher;
+import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TextSearchInfo;
@@ -42,6 +41,8 @@
 import org.elasticsearch.index.mapper.extras.SourceIntervalsSource;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.field.KeywordDocValuesField;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -51,8 +52,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.search.aggregations.support.CoreValuesSourceType.KEYWORD;
-
 public class PatternedTextFieldType extends StringFieldType {
 
     private static final String TEMPLATE_SUFFIX = ".template";
@@ -93,18 +92,19 @@ public String familyTypeName() {
 
     @Override
     public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-        return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH));
+        return SourceValueFetcher.toString(name(), context, format);
     }
 
     private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> getValueFetcherProvider(
         SearchExecutionContext searchExecutionContext
     ) {
         return context -> {
             ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null);
+            SourceProvider sourceProvider = searchExecutionContext.lookup();
             valueFetcher.setNextReader(context);
             return docID -> {
                 try {
-                    return valueFetcher.fetchValues(null, docID, new ArrayList<>());
+                    return valueFetcher.fetchValues(sourceProvider.getSource(context, docID), docID, new ArrayList<>());
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }
@@ -243,17 +243,20 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
 
     @Override
     public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
-        var templateDataBuilder = new SortedSetOrdinalsIndexFieldData.Builder(
-            templateFieldName(),
-            KEYWORD,
-            (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n)
-        );
-        var argsDataBuilder = new SortedSetOrdinalsIndexFieldData.Builder(
-            argsFieldName(),
-            KEYWORD,
-            (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n)
+        if (fieldDataContext.fielddataOperation() != FielddataOperation.SCRIPT) {
+            throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations");
+        }
+        if (textFieldType.isSyntheticSource()) {
+            return new PatternedTextIndexFieldData.Builder(this);
+        }
+        return new SourceValueFetcherSortedBinaryIndexFieldData.Builder(
+            name(),
+            CoreValuesSourceType.KEYWORD,
+            SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())),
+            fieldDataContext.lookupSupplier().get(),
+            KeywordDocValuesField::new
         );
-        return new PatternedTextIndexFieldData.Builder(name(), templateDataBuilder, argsDataBuilder);
+
     }
 
     String templateFieldName() {

x-pack/plugin/mapper-patterned-text/src/test/java/org/elasticsearch/xpack/patternedtext/PatternedTextFieldTypeTests.java

@@ -8,8 +8,6 @@
 package org.elasticsearch.xpack.patternedtext;
 
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
@@ -40,7 +38,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.function.Supplier;
 
 public class PatternedTextFieldTypeTests extends FieldTypeTestCase {
 
@@ -54,8 +51,8 @@ public void testTermsQuery() {
         MappedFieldType ft = new PatternedTextFieldType("field");
         List<BytesRef> terms = new ArrayList<>();
         terms.add(new BytesRef("foo"));
-        terms.add(new BytesRef("bar"));
-        assertEquals(new TermInSetQuery("field", terms), ft.termsQuery(Arrays.asList("foo", "bar"), null));
+        terms.add(new BytesRef("123"));
+        assertEquals(new TermInSetQuery("field", terms), ft.termsQuery(Arrays.asList("foo", "123"), null));
     }
 
     public void testRangeQuery() {
@@ -107,37 +104,25 @@ public void testFuzzyQuery() {
         assertEquals("[fuzzy] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage());
     }
 
-    public void testFetchDocValue() throws IOException {
-        Supplier<Document> documentSupplier = () -> {
-            Document doc = new Document();
-            doc.add(new SortedSetDocValuesField("field.template", new BytesRef("value %W")));
-            doc.add(new SortedSetDocValuesField("field.args", new BytesRef("1")));
-            return doc;
-        };
-
-        MappedFieldType fieldType = new PatternedTextFieldType("field");
-        assertEquals(List.of("value 1"), fetchDocValues(fieldType, documentSupplier));
-    }
-
     private Query unwrapPositionalQuery(Query query) {
         query = ((ConstantScoreQuery) query).getQuery();
         return query;
     }
 
     public void testPhraseQuery() throws IOException {
         MappedFieldType ft = new PatternedTextFieldType("field");
-        TokenStream ts = new CannedTokenStream(new Token("a", 0, 3), new Token("b", 4, 7));
+        TokenStream ts = new CannedTokenStream(new Token("a", 0, 3), new Token("1", 4, 7));
         Query query = ft.phraseQuery(ts, 0, true, MOCK_CONTEXT);
         Query delegate = unwrapPositionalQuery(query);
-        assertEquals(new PhraseQuery("field", "a", "b").toString(), delegate.toString());
+        assertEquals(new PhraseQuery("field", "a", "1").toString(), delegate.toString());
     }
 
     public void testMultiPhraseQuery() throws IOException {
         MappedFieldType ft = new PatternedTextFieldType("field");
-        TokenStream ts = new CannedTokenStream(new Token("a", 0, 3), new Token("b", 0, 0, 3), new Token("c", 4, 7));
+        TokenStream ts = new CannedTokenStream(new Token("a", 0, 3), new Token("2", 0, 0, 3), new Token("c", 4, 7));
         Query query = ft.multiPhraseQuery(ts, 0, true, MOCK_CONTEXT);
         Query delegate = unwrapPositionalQuery(query);
-        Query expected = new MultiPhraseQuery.Builder().add(new Term[] { new Term("field", "a"), new Term("field", "b") })
+        Query expected = new MultiPhraseQuery.Builder().add(new Term[] { new Term("field", "a"), new Term("field", "2") })
             .add(new Term("field", "c"))
             .build();
         assertEquals(expected.toString(), delegate.toString());