Simple version of patterned_text with a single doc value for arguments

modules/mapper-extras/src/main/java/module-info.java

@@ -14,4 +14,6 @@
     requires org.apache.lucene.core;
     requires org.apache.lucene.memory;
     requires org.apache.lucene.queries;
+
+    exports org.elasticsearch.index.mapper.extras;
 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -173,7 +173,7 @@ public MatchOnlyTextFieldType(
             super(name, true, false, false, tsi, meta);
             this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer);
             this.textFieldType = new TextFieldType(name, isSyntheticSource);
-            this.originalName = isSyntheticSource ? name() + "._original" : null;
+            this.originalName = isSyntheticSource ? name + "._original" : null;
         }
 
         public MatchOnlyTextFieldType(String name) {

server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java

@@ -124,7 +124,8 @@ private FieldContext contextBuilders(
                 if (fieldNameContainsWildcards) {
                     if (fieldType.typeName().equals(TextFieldMapper.CONTENT_TYPE) == false
                         && fieldType.typeName().equals(KeywordFieldMapper.CONTENT_TYPE) == false
-                        && fieldType.typeName().equals("match_only_text") == false) {
+                        && fieldType.typeName().equals("match_only_text") == false
+                        && fieldType.typeName().equals("patterned_text") == false) {
                         continue;
                     }
                     if (highlighter.canHighlight(fieldType) == false) {

x-pack/plugin/logsdb/build.gradle

@@ -24,12 +24,13 @@ base {
 
 restResources {
   restApi {
-    include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream', 'ingest', 'cat', 'capabilities', 'esql.query'
+    include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream', 'ingest', 'cat', 'capabilities', 'esql.query', 'field_caps'
   }
 }
 
 dependencies {
   compileOnly project(path: xpackModule('core'))
+  implementation project(':modules:mapper-extras')
   testImplementation project(':modules:data-streams')
   testImplementation(testArtifact(project(xpackModule('core'))))
   javaRestTestImplementation(testArtifact(project(xpackModule('spatial'))))

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java

@@ -12,21 +12,27 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.logsdb.patternedtext.PatternedTextFieldMapper;
+import org.elasticsearch.xpack.logsdb.patternedtext.PatternedTextFieldType;
 
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
+import static java.util.Collections.singletonMap;
 import static org.elasticsearch.xpack.logsdb.LogsdbLicenseService.FALLBACK_SETTING;
 
-public class LogsDBPlugin extends Plugin implements ActionPlugin {
+public class LogsDBPlugin extends Plugin implements ActionPlugin, MapperPlugin {
 
     private final Settings settings;
     private final LogsdbLicenseService licenseService;
@@ -98,6 +104,15 @@ public List<ActionPlugin.ActionHandler> getActions() {
         return actions;
     }
 
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        if (PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled()) {
+            return singletonMap(PatternedTextFieldType.CONTENT_TYPE, PatternedTextFieldMapper.PARSER);
+        } else {
+            return Map.of();
+        }
+    }
+
     protected XPackLicenseState getLicenseState() {
         return XPackPlugin.getSharedLicenseState();
     }

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextDocValues.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb.patternedtext;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+
+public class PatternedTextDocValues extends BinaryDocValues {
+    private final SortedSetDocValues templateDocValues;
+    private final SortedSetDocValues argsDocValues;
+
+    PatternedTextDocValues(SortedSetDocValues templateDocValues, SortedSetDocValues argsDocValues) {
+        this.templateDocValues = templateDocValues;
+        this.argsDocValues = argsDocValues;
+    }
+
+    static PatternedTextDocValues from(LeafReader leafReader, String templateFieldName, String argsFieldName) throws IOException {
+        SortedSetDocValues templateDocValues = DocValues.getSortedSet(leafReader, templateFieldName);
+        if (templateDocValues.getValueCount() == 0) {
+            return null;
+        }
+
+        SortedSetDocValues argsDocValues = DocValues.getSortedSet(leafReader, argsFieldName);
+        return new PatternedTextDocValues(templateDocValues, argsDocValues);
+    }
+
+    private String getNextStringValue() throws IOException {
+        assert templateDocValues.docValueCount() == 1;
+        String template = templateDocValues.lookupOrd(templateDocValues.nextOrd()).utf8ToString();
+        int argsCount = PatternedTextValueProcessor.countArgs(template);
+        if (argsCount > 0) {
+            assert argsDocValues.docValueCount() == 1;
+            var mergedArgs = argsDocValues.lookupOrd(argsDocValues.nextOrd());
+            var args = PatternedTextValueProcessor.decodeRemainingArgs(mergedArgs.utf8ToString());
+            return PatternedTextValueProcessor.merge(new PatternedTextValueProcessor.Parts(template, args));
+        } else {
+            return template;
+        }
+    }
+
+    @Override
+    public BytesRef binaryValue() throws IOException {
+        return new BytesRef(getNextStringValue());
+    }
+
+    @Override
+    public boolean advanceExact(int i) throws IOException {
+        argsDocValues.advanceExact(i);
+        // If template has a value, then message has a value. We don't have to check args here, since there may not be args for the doc
+        return templateDocValues.advanceExact(i);
+    }
+
+    @Override
+    public int docID() {
+        return templateDocValues.docID();
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+        int templateNext = templateDocValues.nextDoc();
+        var argsAdvance = argsDocValues.advance(templateNext);
+        assert argsAdvance >= templateNext;
+        return templateNext;
+    }
+
+    @Override
+    public int advance(int i) throws IOException {
+        int templateAdvance = templateDocValues.advance(i);
+        var argsAdvance = argsDocValues.advance(templateAdvance);
+        assert argsAdvance >= templateAdvance;
+        return templateAdvance;
+    }
+
+    @Override
+    public long cost() {
+        return templateDocValues.cost() + argsDocValues.cost();
+    }
+}

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java

@@ -0,0 +1,176 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb.patternedtext;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.util.FeatureFlag;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
+import org.elasticsearch.index.mapper.DocumentParserContext;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MapperBuilderContext;
+import org.elasticsearch.index.mapper.TextParams;
+import org.elasticsearch.index.mapper.TextSearchInfo;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A {@link FieldMapper} that assigns every document the same value.
+ */
+public class PatternedTextFieldMapper extends FieldMapper {
+
+    public static final FeatureFlag PATTERNED_TEXT_MAPPER = new FeatureFlag("patterned_text");
+
+    public static class Defaults {
+        public static final FieldType FIELD_TYPE;
+
+        static {
+            final FieldType ft = new FieldType();
+            ft.setTokenized(true);
+            ft.setStored(false);
+            ft.setStoreTermVectors(false);
+            ft.setOmitNorms(true);
+            ft.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE = freezeAndDeduplicateFieldType(ft);
+        }
+    }
+
+    public static class Builder extends FieldMapper.Builder {
+
+        private final IndexVersion indexCreatedVersion;
+
+        private final Parameter<Map<String, String>> meta = Parameter.metaParam();
+
+        private final TextParams.Analyzers analyzers;
+
+        public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
+            super(name);
+            this.indexCreatedVersion = indexCreatedVersion;
+            this.analyzers = new TextParams.Analyzers(
+                indexAnalyzers,
+                m -> ((PatternedTextFieldMapper) m).indexAnalyzer,
+                m -> ((PatternedTextFieldMapper) m).positionIncrementGap,
+                indexCreatedVersion
+            );
+        }
+
+        @Override
+        protected Parameter<?>[] getParameters() {
+            return new Parameter<?>[] { meta };
+        }
+
+        private PatternedTextFieldType buildFieldType(MapperBuilderContext context) {
+            NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer();
+            NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer();
+            NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer();
+            TextSearchInfo tsi = new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchQuoteAnalyzer);
+            return new PatternedTextFieldType(
+                context.buildFullName(leafName()),
+                tsi,
+                indexAnalyzer,
+                context.isSourceSynthetic(),
+                meta.getValue()
+            );
+        }
+
+        @Override
+        public PatternedTextFieldMapper build(MapperBuilderContext context) {
+            return new PatternedTextFieldMapper(leafName(), buildFieldType(context), builderParams(this, context), this);
+        }
+    }
+
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
+
+    private final IndexVersion indexCreatedVersion;
+    private final IndexAnalyzers indexAnalyzers;
+    private final NamedAnalyzer indexAnalyzer;
+    private final int positionIncrementGap;
+    private final FieldType fieldType;
+
+    private PatternedTextFieldMapper(
+        String simpleName,
+        PatternedTextFieldType mappedFieldPatternedTextFieldType,
+        BuilderParams builderParams,
+        Builder builder
+    ) {
+        super(simpleName, mappedFieldPatternedTextFieldType, builderParams);
+        assert mappedFieldPatternedTextFieldType.getTextSearchInfo().isTokenized();
+        assert mappedFieldPatternedTextFieldType.hasDocValues() == false;
+        this.fieldType = Defaults.FIELD_TYPE;
+        this.indexCreatedVersion = builder.indexCreatedVersion;
+        this.indexAnalyzers = builder.analyzers.indexAnalyzers;
+        this.indexAnalyzer = builder.analyzers.getIndexAnalyzer();
+        this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
+    }
+
+    @Override
+    public Map<String, NamedAnalyzer> indexAnalyzers() {
+        return Map.of(mappedFieldType.name(), indexAnalyzer);
+    }
+
+    @Override
+    public FieldMapper.Builder getMergeBuilder() {
+        return new Builder(leafName(), indexCreatedVersion, indexAnalyzers).init(this);
+    }
+
+    @Override
+    protected void parseCreateField(DocumentParserContext context) throws IOException {
+        final String value = context.parser().textOrNull();
+        if (value == null) {
+            return;
+        }
+
+        var existingValue = context.doc().getField(fieldType().name());
+        if (existingValue != null) {
+            throw new IllegalArgumentException("Multiple values are not allowed for field [" + fieldType().name() + "].");
+        }
+
+        // Parse template and args.
+        PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value);
+
+        // Add index on original value
+        context.doc().add(new Field(fieldType().name(), value, fieldType));
+
+        // Add template doc_values
+        context.doc().add(new SortedSetDocValuesField(fieldType().templateFieldName(), new BytesRef(parts.template())));
+
+        // Add args doc_values
+        if (parts.args().isEmpty() == false) {
+            String remainingArgs = PatternedTextValueProcessor.encodeRemainingArgs(parts);
+            context.doc().add(new SortedSetDocValuesField(fieldType().argsFieldName(), new BytesRef(remainingArgs)));
+        }
+    }
+
+    @Override
+    protected String contentType() {
+        return PatternedTextFieldType.CONTENT_TYPE;
+    }
+
+    @Override
+    public PatternedTextFieldType fieldType() {
+        return (PatternedTextFieldType) super.fieldType();
+    }
+
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupport() {
+        return new SyntheticSourceSupport.Native(
+            () -> new CompositeSyntheticFieldLoader(
+                leafName(),
+                fullPath(),
+                new PatternedTextSyntheticFieldLoaderLayer(fieldType().name(), fieldType().templateFieldName(), fieldType().argsFieldName())
+            )
+        );
+    }
+}