Add template_id to patterned-text type (elastic#131401)

For patterned-text mapper foo, add a sub-field called foo.template_id. This is the an 8-byte hash of the template doc_value column. Unlike the template, template_id is accessible and can be used for querying, aggregations, etc. The template_id is stored as a KeywordField and can use any features of the KeyworkFieldType. template_id has doc_values, and is not stored or indexed. It uses doc value skippers, which should be quite fast given that the index will be sorted on template.

Author: parkertimmins

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -209,7 +209,8 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
         private final IndexAnalyzers indexAnalyzers;
         private final ScriptCompiler scriptCompiler;
         private final IndexVersion indexCreatedVersion;
-        private final boolean useDocValuesSkipper;
+        private final boolean enableDocValuesSkipper;
+        private final boolean forceDocValuesSkipper;
         private final SourceKeepMode indexSourceKeepMode;
 
         public Builder(final String name, final MappingParserContext mappingParserContext) {
@@ -222,6 +223,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
                 mappingParserContext.getIndexSettings().getMode(),
                 mappingParserContext.getIndexSettings().getIndexSortConfig(),
                 USE_DOC_VALUES_SKIPPER.get(mappingParserContext.getSettings()),
+                false,
                 mappingParserContext.getIndexSettings().sourceKeepMode()
             );
         }
@@ -243,6 +245,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
                 IndexMode.STANDARD,
                 null,
                 false,
+                false,
                 sourceKeepMode
             );
         }
@@ -255,7 +258,8 @@ private Builder(
             IndexVersion indexCreatedVersion,
             IndexMode indexMode,
             IndexSortConfig indexSortConfig,
-            boolean useDocValuesSkipper,
+            boolean enableDocValuesSkipper,
+            boolean forceDocValuesSkipper,
             SourceKeepMode indexSourceKeepMode
         ) {
             super(name);
@@ -293,14 +297,36 @@ private Builder(
                 });
             this.indexSortConfig = indexSortConfig;
             this.indexMode = indexMode;
-            this.useDocValuesSkipper = useDocValuesSkipper;
+            this.enableDocValuesSkipper = enableDocValuesSkipper;
+            this.forceDocValuesSkipper = forceDocValuesSkipper;
             this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         public Builder(String name, IndexVersion indexCreatedVersion) {
             this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE);
         }
 
+        public static Builder buildWithDocValuesSkipper(
+            String name,
+            IndexMode indexMode,
+            IndexVersion indexCreatedVersion,
+            boolean enableDocValuesSkipper
+        ) {
+            return new Builder(
+                name,
+                null,
+                ScriptCompiler.NONE,
+                Integer.MAX_VALUE,
+                indexCreatedVersion,
+                indexMode,
+                // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed.
+                null,
+                enableDocValuesSkipper,
+                true,
+                SourceKeepMode.NONE
+            );
+        }
+
         public Builder ignoreAbove(int ignoreAbove) {
             this.ignoreAbove.setValue(ignoreAbove);
             return this;
@@ -422,7 +448,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType
         @Override
         public KeywordFieldMapper build(MapperBuilderContext context) {
             FieldType fieldtype = resolveFieldType(
-                useDocValuesSkipper,
+                enableDocValuesSkipper,
+                forceDocValuesSkipper,
+                hasDocValues,
                 indexCreatedVersion,
                 indexSortConfig,
                 indexMode,
@@ -460,24 +488,30 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
                 buildFieldType(context, fieldtype),
                 builderParams(this, context),
                 context.isSourceSynthetic(),
-                useDocValuesSkipper,
                 this,
                 offsetsFieldName,
                 indexSourceKeepMode
             );
         }
 
-        private FieldType resolveFieldType(
-            final boolean useDocValuesSkipper,
+        private static FieldType resolveFieldType(
+            final boolean enableDocValuesSkipper,
+            final boolean forceDocValuesSkipper,
+            final Parameter<Boolean> hasDocValues,
             final IndexVersion indexCreatedVersion,
             final IndexSortConfig indexSortConfig,
             final IndexMode indexMode,
             final String fullFieldName
         ) {
-            if (useDocValuesSkipper
-                && indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX)
-                && shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) {
-                return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
+            if (enableDocValuesSkipper) {
+                if (forceDocValuesSkipper) {
+                    assert hasDocValues.getValue();
+                    return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
+                }
+                if (indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX)
+                    && shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) {
+                    return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
+                }
             }
             return new FieldType(Defaults.FIELD_TYPE);
         }
@@ -1088,7 +1122,8 @@ public String originalName() {
     private final int ignoreAboveDefault;
     private final IndexMode indexMode;
     private final IndexSortConfig indexSortConfig;
-    private final boolean useDocValuesSkipper;
+    private final boolean enableDocValuesSkipper;
+    private final boolean forceDocValuesSkipper;
     private final String offsetsFieldName;
     private final SourceKeepMode indexSourceKeepMode;
     private final String originalName;
@@ -1099,7 +1134,6 @@ private KeywordFieldMapper(
         KeywordFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean isSyntheticSource,
-        boolean useDocValuesSkipper,
         Builder builder,
         String offsetsFieldName,
         SourceKeepMode indexSourceKeepMode
@@ -1120,7 +1154,8 @@ private KeywordFieldMapper(
         this.ignoreAboveDefault = builder.ignoreAboveDefault;
         this.indexMode = builder.indexMode;
         this.indexSortConfig = builder.indexSortConfig;
-        this.useDocValuesSkipper = useDocValuesSkipper;
+        this.enableDocValuesSkipper = builder.enableDocValuesSkipper;
+        this.forceDocValuesSkipper = builder.forceDocValuesSkipper;
         this.offsetsFieldName = offsetsFieldName;
         this.indexSourceKeepMode = indexSourceKeepMode;
         this.originalName = mappedFieldType.originalName();
@@ -1219,7 +1254,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
             throw new IllegalArgumentException(msg);
         }
 
-        Field field = new KeywordField(fieldType().name(), binaryValue, fieldType);
+        Field field = buildKeywordField(binaryValue);
         context.doc().add(field);
 
         if (fieldType().hasDocValues() == false && fieldType.omitNorms()) {
@@ -1276,11 +1311,16 @@ public FieldMapper.Builder getMergeBuilder() {
             indexCreatedVersion,
             indexMode,
             indexSortConfig,
-            useDocValuesSkipper,
+            enableDocValuesSkipper,
+            forceDocValuesSkipper,
             indexSourceKeepMode
         ).dimension(fieldType().isDimension()).init(this);
     }
 
+    public Field buildKeywordField(BytesRef binaryValue) {
+        return new KeywordField(fieldType().name(), binaryValue, fieldType);
+    }
+
     @Override
     public void doValidate(MappingLookup lookup) {
         if (fieldType().isDimension() && null != lookup.nestedLookup().getNestedParent(fullPath())) {

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java

@@ -13,19 +13,28 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.util.FeatureFlag;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
+import org.elasticsearch.index.mapper.MappingParserContext;
 import org.elasticsearch.index.mapper.TextParams;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
+
 /**
  * A {@link FieldMapper} that assigns every document the same value.
  */
@@ -50,20 +59,38 @@ public static class Defaults {
     public static class Builder extends FieldMapper.Builder {
 
         private final IndexVersion indexCreatedVersion;
-
+        private final IndexSettings indexSettings;
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
-
         private final TextParams.Analyzers analyzers;
+        private final boolean enableDocValuesSkipper;
+
+        public Builder(String name, MappingParserContext context) {
+            this(
+                name,
+                context.indexVersionCreated(),
+                context.getIndexSettings(),
+                context.getIndexAnalyzers(),
+                USE_DOC_VALUES_SKIPPER.get(context.getSettings())
+            );
+        }
 
-        public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
+        public Builder(
+            String name,
+            IndexVersion indexCreatedVersion,
+            IndexSettings indexSettings,
+            IndexAnalyzers indexAnalyzers,
+            boolean enableDocValuesSkipper
+        ) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
+            this.indexSettings = indexSettings;
             this.analyzers = new TextParams.Analyzers(
                 indexAnalyzers,
                 m -> ((PatternedTextFieldMapper) m).indexAnalyzer,
                 m -> ((PatternedTextFieldMapper) m).positionIncrementGap,
                 indexCreatedVersion
             );
+            this.enableDocValuesSkipper = enableDocValuesSkipper;
         }
 
         @Override
@@ -87,23 +114,35 @@ private PatternedTextFieldType buildFieldType(MapperBuilderContext context) {
 
         @Override
         public PatternedTextFieldMapper build(MapperBuilderContext context) {
-            return new PatternedTextFieldMapper(leafName(), buildFieldType(context), builderParams(this, context), this);
+            PatternedTextFieldType patternedTextFieldType = buildFieldType(context);
+            BuilderParams builderParams = builderParams(this, context);
+            var templateIdMapper = KeywordFieldMapper.Builder.buildWithDocValuesSkipper(
+                patternedTextFieldType.templateIdFieldName(),
+                indexSettings.getMode(),
+                indexCreatedVersion,
+                enableDocValuesSkipper
+            ).build(context);
+            return new PatternedTextFieldMapper(leafName(), patternedTextFieldType, builderParams, this, templateIdMapper);
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
+    public static final TypeParser PARSER = new TypeParser(Builder::new);
 
     private final IndexVersion indexCreatedVersion;
     private final IndexAnalyzers indexAnalyzers;
+    private final IndexSettings indexSettings;
     private final NamedAnalyzer indexAnalyzer;
+    private final boolean enableDocValuesSkipper;
     private final int positionIncrementGap;
     private final FieldType fieldType;
+    private final KeywordFieldMapper templateIdMapper;
 
     private PatternedTextFieldMapper(
         String simpleName,
         PatternedTextFieldType mappedFieldPatternedTextFieldType,
         BuilderParams builderParams,
-        Builder builder
+        Builder builder,
+        KeywordFieldMapper templateIdMapper
     ) {
         super(simpleName, mappedFieldPatternedTextFieldType, builderParams);
         assert mappedFieldPatternedTextFieldType.getTextSearchInfo().isTokenized();
@@ -112,7 +151,10 @@ private PatternedTextFieldMapper(
         this.indexCreatedVersion = builder.indexCreatedVersion;
         this.indexAnalyzers = builder.analyzers.indexAnalyzers;
         this.indexAnalyzer = builder.analyzers.getIndexAnalyzer();
+        this.indexSettings = builder.indexSettings;
+        this.enableDocValuesSkipper = builder.enableDocValuesSkipper;
         this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
+        this.templateIdMapper = templateIdMapper;
     }
 
     @Override
@@ -122,7 +164,18 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexCreatedVersion, indexAnalyzers).init(this);
+        return new Builder(leafName(), indexCreatedVersion, indexSettings, indexAnalyzers, enableDocValuesSkipper).init(this);
+    }
+
+    @Override
+    public Iterator<Mapper> iterator() {
+        List<Mapper> mappers = new ArrayList<>();
+        Iterator<Mapper> m = super.iterator();
+        while (m.hasNext()) {
+            mappers.add(m.next());
+        }
+        mappers.add(templateIdMapper);
+        return mappers.iterator();
     }
 
     @Override
@@ -146,6 +199,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         // Add template doc_values
         context.doc().add(new SortedSetDocValuesField(fieldType().templateFieldName(), new BytesRef(parts.template())));
 
+        // Add template_id doc_values
+        context.doc().add(templateIdMapper.buildKeywordField(new BytesRef(parts.templateId())));
+
         // Add args doc_values
         if (parts.args().isEmpty() == false) {
             String remainingArgs = PatternedTextValueProcessor.encodeRemainingArgs(parts);

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java

@@ -55,6 +55,7 @@
 public class PatternedTextFieldType extends StringFieldType {
 
     private static final String TEMPLATE_SUFFIX = ".template";
+    private static final String TEMPLATE_ID_SUFFIX = ".template_id";
     private static final String ARGS_SUFFIX = ".args";
 
     public static final String CONTENT_TYPE = "patterned_text";
@@ -145,7 +146,7 @@ public Query fuzzyQuery(
 
     @Override
     public Query existsQuery(SearchExecutionContext context) {
-        return new FieldExistsQuery(templateFieldName());
+        return new FieldExistsQuery(templateIdFieldName());
     }
 
     @Override
@@ -263,6 +264,10 @@ String templateFieldName() {
         return name() + TEMPLATE_SUFFIX;
     }
 
+    String templateIdFieldName() {
+        return name() + TEMPLATE_ID_SUFFIX;
+    }
+
     String argsFieldName() {
         return name() + ARGS_SUFFIX;
     }

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessor.java

@@ -7,6 +7,11 @@
 
 package org.elasticsearch.xpack.logsdb.patternedtext;
 
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.hash.MurmurHash3;
+import org.elasticsearch.common.util.ByteUtils;
+
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -16,7 +21,20 @@ public class PatternedTextValueProcessor {
     private static final String DELIMITER = "[\\s\\[\\]]";
     private static final String SPACE = " ";
 
-    record Parts(String template, List<String> args) {}
+    record Parts(String template, String templateId, List<String> args) {
+        Parts(String template, List<String> args) {
+            this(template, PatternedTextValueProcessor.templateId(template), args);
+        }
+    }
+
+    static String templateId(String template) {
+        byte[] bytes = template.getBytes(StandardCharsets.UTF_8);
+        MurmurHash3.Hash128 hash = new MurmurHash3.Hash128();
+        MurmurHash3.hash128(bytes, 0, bytes.length, 0, hash);
+        byte[] hashBytes = new byte[8];
+        ByteUtils.writeLongLE(hash.h1, hashBytes, 0);
+        return Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(hashBytes);
+    }
 
     static Parts split(String text) {
         StringBuilder template = new StringBuilder();