Patterned-text field prototype

benchmarks/build.gradle

@@ -45,6 +45,7 @@ dependencies {
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
   implementation project(path: ':libs:simdvec')
+  implementation project(':x-pack:plugin:mapper-patterned-text')
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))
   nativeLib(project(':libs:native'))

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/PatternedTextMapperOperationsBenchmark.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.index.mapper;
+
+//import org.elasticsearch.xpack.patternedtext.PatternedTextValueProcessor;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(value = 1)
+@Warmup(iterations = 2)
+@Measurement(iterations = 3)
+@State(Scope.Benchmark)
+public class PatternedTextMapperOperationsBenchmark {
+    @Param({})
+    public String input;
+
+    @Benchmark
+    public void testIpv4MatchManual(Blackhole blackhole) {}
+}

modules/mapper-extras/src/main/java/module-info.java

@@ -14,4 +14,6 @@
     requires org.apache.lucene.core;
     requires org.apache.lucene.memory;
     requires org.apache.lucene.queries;
+
+    exports org.elasticsearch.index.mapper.extras;
 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -169,7 +169,7 @@ public MatchOnlyTextFieldType(
             super(name, true, false, false, tsi, meta);
             this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer);
             this.textFieldType = new TextFieldType(name, isSyntheticSource);
-            this.originalName = isSyntheticSource ? name() + "._original" : null;
+            this.originalName = isSyntheticSource ? name + "._original" : null;
         }
 
         public MatchOnlyTextFieldType(String name) {

server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java

@@ -549,10 +549,10 @@ public String toString() {
         }
     }
 
-    private static class SingletonOrdinals extends BlockDocValuesReader {
+    public static class SingletonOrdinals extends BlockDocValuesReader {
         private final SortedDocValues ordinals;
 
-        SingletonOrdinals(SortedDocValues ordinals) {
+        public SingletonOrdinals(SortedDocValues ordinals) {
             this.ordinals = ordinals;
         }
 
@@ -607,10 +607,10 @@ public String toString() {
         }
     }
 
-    private static class Ordinals extends BlockDocValuesReader {
+    public static class Ordinals extends BlockDocValuesReader {
         private final SortedSetDocValues ordinals;
 
-        Ordinals(SortedSetDocValues ordinals) {
+        public Ordinals(SortedSetDocValues ordinals) {
             this.ordinals = ordinals;
         }
 

server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java

@@ -124,7 +124,8 @@ private FieldContext contextBuilders(
                 if (fieldNameContainsWildcards) {
                     if (fieldType.typeName().equals(TextFieldMapper.CONTENT_TYPE) == false
                         && fieldType.typeName().equals(KeywordFieldMapper.CONTENT_TYPE) == false
-                        && fieldType.typeName().equals("match_only_text") == false) {
+                        && fieldType.typeName().equals("match_only_text") == false
+                        && fieldType.typeName().equals("patterned_text") == false) {
                         continue;
                     }
                     if (highlighter.canHighlight(fieldType) == false) {

test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java

@@ -8,14 +8,21 @@
  */
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesSkipIndexType;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.FieldLookup;
@@ -32,6 +39,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Supplier;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -71,6 +79,24 @@ public static List<?> fetchSourceValue(MappedFieldType fieldType, Object sourceV
         return fetcher.fetchValues(source, -1, new ArrayList<>());
     }
 
+    public static List<?> fetchDocValues(MappedFieldType fieldType, Supplier<Document> documentSupplier) throws IOException {
+        IndexWriterConfig iwc = new IndexWriterConfig(null);
+        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc)) {
+            iw.addDocument(documentSupplier.get());
+            try (DirectoryReader reader = iw.getReader()) {
+                IndexSearcher searcher = newSearcher(reader);
+                LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+                SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
+                when(searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH)).thenReturn(
+                    fieldType.fielddataBuilder(null).build(null, null)
+                );
+                ValueFetcher valueFetcher = fieldType.valueFetcher(searchExecutionContext, null);
+                valueFetcher.setNextReader(context);
+                return valueFetcher.fetchValues(null, 0, new ArrayList<>());
+            }
+        }
+    }
+
     public static List<?> fetchSourceValues(MappedFieldType fieldType, Object... values) throws IOException {
         String field = fieldType.name();
         SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);

x-pack/plugin/mapper-patterned-text/build.gradle

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+
+esplugin {
+  name = 'patterned-text'
+  description = 'Module for the patterned_text field type.'
+  classname ='org.elasticsearch.xpack.patternedtext.PatternedTextMapperPlugin'
+  extendedPlugins = ['x-pack-core', 'lang-painless']
+}
+base {
+  archivesName = 'x-pack-patterned-text'
+}
+
+dependencies {
+  compileOnly project(':modules:lang-painless:spi')
+  compileOnly project(path: xpackModule('core'))
+  implementation project(':modules:mapper-extras')
+}
+
+if (buildParams.getSnapshotBuild() == false) {
+  tasks.named("test").configure {
+    systemProperty 'es.index_mode_feature_flag_registered', 'true'
+  }
+}

x-pack/plugin/mapper-patterned-text/src/main/java/org/elasticsearch/xpack/patternedtext/PatternedTextBlockLoader.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.patternedtext;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.elasticsearch.index.mapper.BlockDocValuesReader;
+
+import java.io.IOException;
+
+public class PatternedTextBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
+
+    private final String name;
+    private final String templateFieldName;
+    private final String timestampFieldName;
+    private final String argsFieldName;
+
+    PatternedTextBlockLoader(String name, String templateFieldName, String timestampFieldName, String argsFieldName) {
+        this.name = name;
+        this.templateFieldName = templateFieldName;
+        this.timestampFieldName = timestampFieldName;
+        this.argsFieldName = argsFieldName;
+    }
+
+    @Override
+    public BytesRefBuilder builder(BlockFactory factory, int expectedCount) {
+        return factory.bytesRefs(expectedCount);
+    }
+
+    @Override
+    public AllReader reader(LeafReaderContext context) throws IOException {
+        SortedSetDocValues combinedDocValues = ordinals(context);
+        if (combinedDocValues != null) {
+            SortedDocValues singleton = DocValues.unwrapSingleton(combinedDocValues);
+            if (singleton != null) {
+                return new BlockDocValuesReader.SingletonOrdinals(singleton);
+            }
+            return new BlockDocValuesReader.Ordinals(combinedDocValues);
+        }
+        return new ConstantNullsReader();
+    }
+
+    @Override
+    public boolean supportsOrdinals() {
+        return true;
+    }
+
+    @Override
+    public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException {
+        return PatternedTextDocValues.from(context.reader(), templateFieldName, timestampFieldName, argsFieldName);
+    }
+
+    @Override
+    public String toString() {
+        return "BytesRefsFromOrds[" + name + "]";
+    }
+}

x-pack/plugin/mapper-patterned-text/src/main/java/org/elasticsearch/xpack/patternedtext/PatternedTextDocValues.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.patternedtext;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class PatternedTextDocValues extends SortedSetDocValues {
+    private final SortedSetDocValues templateDocValues;
+    private final SortedSetDocValues argsDocValues;
+    private final SortedSetDocValues[] optimizedArgsDocValues;
+    private final SortedNumericDocValues timestampDocValues;
+
+    PatternedTextDocValues(
+        SortedSetDocValues templateDocValues,
+        SortedSetDocValues argsDocValues,
+        SortedSetDocValues[] optimizedArgsDocValues,
+        SortedNumericDocValues timestampDocValues
+    ) {
+        this.templateDocValues = templateDocValues;
+        this.argsDocValues = argsDocValues;
+        this.optimizedArgsDocValues = optimizedArgsDocValues;
+        this.timestampDocValues = timestampDocValues;
+    }
+
+    static PatternedTextDocValues from(LeafReader leafReader, String templateFieldName, String timestampFieldName, String argsFieldName)
+        throws IOException {
+        SortedSetDocValues templateDocValues = DocValues.getSortedSet(leafReader, templateFieldName);
+        if (templateDocValues.getValueCount() == 0) {
+            return null;
+        }
+
+        SortedSetDocValues argsDocValues = DocValues.getSortedSet(leafReader, argsFieldName);
+        SortedSetDocValues[] optimizedArgsDocValues = new SortedSetDocValues[PatternedTextFieldMapper.OPTIMIZED_ARG_COUNT];
+        for (int i = 0; i < optimizedArgsDocValues.length; i++) {
+            optimizedArgsDocValues[i] = DocValues.getSortedSet(leafReader, argsFieldName + "." + i);
+        }
+        SortedNumericDocValues timestampDocValues = DocValues.getSortedNumeric(leafReader, timestampFieldName);
+        return new PatternedTextDocValues(templateDocValues, argsDocValues, optimizedArgsDocValues, timestampDocValues);
+    }
+
+    @Override
+    public long nextOrd() throws IOException {
+        return templateDocValues.nextOrd();
+    }
+
+    @Override
+    public int docValueCount() {
+        return templateDocValues.docValueCount();
+    }
+
+    @Override
+    public BytesRef lookupOrd(long l) throws IOException {
+        return new BytesRef(lookupOrdAsString(l));
+    }
+
+    String lookupOrdAsString(long l) throws IOException {
+        String template = templateDocValues.lookupOrd(l).utf8ToString();
+        Long timestamp = PatternedTextValueProcessor.hasTimestamp(template) ? timestampDocValues.nextValue() : null;
+
+        int argsCount = PatternedTextValueProcessor.countArgs(template);
+        List<String> args = new ArrayList<>(argsCount);
+        for (int j = 0; j < Integer.min(argsCount, PatternedTextFieldMapper.OPTIMIZED_ARG_COUNT); j++) {
+            args.add(optimizedArgsDocValues[j].lookupOrd(argsDocValues.nextOrd()).utf8ToString());
+        }
+        if (argsCount > PatternedTextFieldMapper.OPTIMIZED_ARG_COUNT) {
+            PatternedTextValueProcessor.addRemainingArgs(args, argsDocValues.lookupOrd(argsDocValues.nextOrd()).utf8ToString());
+        }
+        return PatternedTextValueProcessor.merge(new PatternedTextValueProcessor.Parts(template, timestamp, args, null));
+    }
+
+    @Override
+    public long getValueCount() {
+        return templateDocValues.getValueCount();
+    }
+
+    @Override
+    public boolean advanceExact(int i) throws IOException {
+        timestampDocValues.advanceExact(i);
+        argsDocValues.advanceExact(i);
+        for (var optimizedArg : optimizedArgsDocValues) {
+            optimizedArg.advanceExact(i);
+        }
+        return templateDocValues.advanceExact(i);
+    }
+
+    @Override
+    public int docID() {
+        return templateDocValues.docID();
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+        return templateDocValues.nextDoc();
+    }
+
+    @Override
+    public int advance(int i) throws IOException {
+        return templateDocValues.advance(i);
+    }
+
+    @Override
+    public long cost() {
+        return templateDocValues.cost() + argsDocValues.cost() + timestampDocValues.cost();
+    }
+}

x-pack/plugin/mapper-patterned-text/src/main/java/org/elasticsearch/xpack/patternedtext/PatternedTextDocValuesField.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.patternedtext;
+
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.script.field.BaseKeywordDocValuesField;
+
+public class PatternedTextDocValuesField extends BaseKeywordDocValuesField {
+    public PatternedTextDocValuesField(SortedBinaryDocValues input, String name) {
+        super(input, name);
+    }
+}