Change patterned_text analyzer from standard to delimiter. (elastic#134434)

martijnvg · gmjehovich · commit 5ab54508c900 · 2025-09-18T10:18:20.000-05:00
Delimiter analyzer splits text based a pre-defined list of delimiters.
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/DelimiterAnalyzer.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/DelimiterAnalyzer.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb.patternedtext;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.pattern.PatternTokenizer;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+
+import java.util.regex.Pattern;
+
+/**
+ * An analyzer that tokenizes text by a pre-defined list of delimiters that work well for log messages.
+ * The pre-defined list of delimiters is: whitespace characters, =, ?, :, [, ], {, }, ", \, '
+ */
+public final class DelimiterAnalyzer extends Analyzer {
+
+    static final NamedAnalyzer INSTANCE = new NamedAnalyzer("delimiter", AnalyzerScope.GLOBAL, new DelimiterAnalyzer());
+
+    private final Pattern pattern;
+
+    private DelimiterAnalyzer() {
+        this.pattern = Regex.compile("[\\s\\=\\?\\:\\[\\]\\{\\}\\\"\\\\\\']", null);
+    }
+
+    @Override
+    protected TokenStreamComponents createComponents(String s) {
+        final Tokenizer tokenizer = new PatternTokenizer(pattern, -1);
+        TokenStream stream = new LowerCaseFilter(tokenizer);
+        return new TokenStreamComponents(tokenizer, stream);
+    }
+
+    @Override
+    protected TokenStream normalize(String fieldName, TokenStream in) {
+        TokenStream stream = in;
+        stream = new LowerCaseFilter(stream);
+        return stream;
+    }
+}
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.logsdb.patternedtext;
 
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedSetDocValuesField;
@@ -18,7 +19,7 @@
 import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
@@ -46,6 +47,7 @@
 public class PatternedTextFieldMapper extends FieldMapper {
 
     public static final FeatureFlag PATTERNED_TEXT_MAPPER = new FeatureFlag("patterned_text");
+    private static final NamedAnalyzer STANDARD_ANALYZER = new NamedAnalyzer("standard", AnalyzerScope.GLOBAL, new StandardAnalyzer());
 
     public static class Defaults {
         public static final FieldType FIELD_TYPE_DOCS;
@@ -77,39 +79,32 @@ public static class Builder extends FieldMapper.Builder {
         private final IndexVersion indexCreatedVersion;
         private final IndexSettings indexSettings;
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
-        private final TextParams.Analyzers analyzers;
         private final Parameter<String> indexOptions = patternedTextIndexOptions(m -> ((PatternedTextFieldMapper) m).indexOptions);
+        private final Parameter<NamedAnalyzer> analyzer;
 
         public Builder(String name, MappingParserContext context) {
-            this(name, context.indexVersionCreated(), context.getIndexSettings(), context.getIndexAnalyzers());
+            this(name, context.indexVersionCreated(), context.getIndexSettings());
         }
 
-        public Builder(String name, IndexVersion indexCreatedVersion, IndexSettings indexSettings, IndexAnalyzers indexAnalyzers) {
+        public Builder(String name, IndexVersion indexCreatedVersion, IndexSettings indexSettings) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
             this.indexSettings = indexSettings;
-            this.analyzers = new TextParams.Analyzers(
-                indexAnalyzers,
-                m -> ((PatternedTextFieldMapper) m).indexAnalyzer,
-                m -> ((PatternedTextFieldMapper) m).positionIncrementGap,
-                indexCreatedVersion
-            );
+            this.analyzer = analyzerParam(name, m -> ((PatternedTextFieldMapper) m).analyzer);
         }
 
         @Override
         protected Parameter<?>[] getParameters() {
-            return new Parameter<?>[] { meta, indexOptions };
+            return new Parameter<?>[] { meta, indexOptions, analyzer };
         }
 
         private PatternedTextFieldType buildFieldType(FieldType fieldType, MapperBuilderContext context) {
-            NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer();
-            NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer();
-            NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer();
-            TextSearchInfo tsi = new TextSearchInfo(fieldType, null, searchAnalyzer, searchQuoteAnalyzer);
+            NamedAnalyzer analyzer = this.analyzer.get();
+            TextSearchInfo tsi = new TextSearchInfo(fieldType, null, analyzer, analyzer);
             return new PatternedTextFieldType(
                 context.buildFullName(leafName()),
                 tsi,
-                indexAnalyzer,
+                analyzer,
                 context.isSourceSynthetic(),
                 meta.getValue()
             );
@@ -134,6 +129,22 @@ private static Parameter<String> patternedTextIndexOptions(Function<FieldMapper,
             });
         }
 
+        private static Parameter<NamedAnalyzer> analyzerParam(String name, Function<FieldMapper, NamedAnalyzer> initializer) {
+            return new Parameter<>("analyzer", false, () -> DelimiterAnalyzer.INSTANCE, (n, c, o) -> {
+                String analyzerName = o.toString();
+                switch (analyzerName) {
+                    case "standard":
+                        return STANDARD_ANALYZER;
+                    case "delimiter":
+                        return DelimiterAnalyzer.INSTANCE;
+                    default:
+                        throw new IllegalArgumentException(
+                            "unsupported analyzer [" + analyzerName + "] for field [" + name + "], supported analyzers are [standard, log]"
+                        );
+                }
+            }, initializer, (b, n, v) -> b.field(n, v.name()), NamedAnalyzer::name);
+        }
+
         @Override
         public PatternedTextFieldMapper build(MapperBuilderContext context) {
             FieldType fieldType = buildLuceneFieldType(indexOptions);
@@ -152,11 +163,9 @@ public PatternedTextFieldMapper build(MapperBuilderContext context) {
     public static final TypeParser PARSER = new TypeParser(Builder::new);
 
     private final IndexVersion indexCreatedVersion;
-    private final IndexAnalyzers indexAnalyzers;
-    private final NamedAnalyzer indexAnalyzer;
+    private final NamedAnalyzer analyzer;
     private final IndexSettings indexSettings;
     private final String indexOptions;
-    private final int positionIncrementGap;
     private final FieldType fieldType;
     private final KeywordFieldMapper templateIdMapper;
 
@@ -173,22 +182,20 @@ private PatternedTextFieldMapper(
         assert mappedFieldType.hasDocValues() == false;
         this.fieldType = fieldType;
         this.indexCreatedVersion = builder.indexCreatedVersion;
-        this.indexAnalyzers = builder.analyzers.indexAnalyzers;
-        this.indexAnalyzer = builder.analyzers.getIndexAnalyzer();
+        this.analyzer = builder.analyzer.get();
         this.indexSettings = builder.indexSettings;
         this.indexOptions = builder.indexOptions.getValue();
-        this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
         this.templateIdMapper = templateIdMapper;
     }
 
     @Override
     public Map<String, NamedAnalyzer> indexAnalyzers() {
-        return Map.of(mappedFieldType.name(), indexAnalyzer);
+        return Map.of(mappedFieldType.name(), analyzer);
     }
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexCreatedVersion, indexSettings, indexAnalyzers).init(this);
+        return new Builder(leafName(), indexCreatedVersion, indexSettings).init(this);
     }
 
     @Override
@@ -269,4 +276,8 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
             )
         );
     }
+
+    NamedAnalyzer getAnalyzer() {
+        return analyzer;
+    }
 }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java
@@ -25,7 +25,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOFunction;
 import org.elasticsearch.common.CheckedIntFunction;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -81,10 +80,10 @@ public class PatternedTextFieldType extends StringFieldType {
             new TextSearchInfo(
                 hasPositions ? PatternedTextFieldMapper.Defaults.FIELD_TYPE_POSITIONS : PatternedTextFieldMapper.Defaults.FIELD_TYPE_DOCS,
                 null,
-                Lucene.STANDARD_ANALYZER,
-                Lucene.STANDARD_ANALYZER
+                DelimiterAnalyzer.INSTANCE,
+                DelimiterAnalyzer.INSTANCE
             ),
-            Lucene.STANDARD_ANALYZER,
+            DelimiterAnalyzer.INSTANCE,
             syntheticSource,
             Collections.emptyMap()
         );
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java
@@ -22,7 +22,9 @@
 import org.apache.lucene.tests.analysis.Token;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -35,8 +37,11 @@
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.logsdb.LogsDBPlugin;
 import org.junit.AssumptionViolatedException;
 import org.junit.Before;
@@ -45,6 +50,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -276,6 +282,49 @@ public void testDocValuesSynthetic() throws IOException {
         assertScriptDocValues(mapper, "foo", equalTo(List.of("foo")));
     }
 
+    public void testAnalyzerAttributeDefault() throws IOException {
+        MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "patterned_text")));
+        var fieldMapper = (PatternedTextFieldMapper) mapper.mappingLookup().getMapper("field");
+        XContentBuilder builder = JsonXContent.contentBuilder().startObject();
+        fieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        var result = (Map<?, ?>) XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2().get("field");
+        assertThat(result.size(), equalTo(1));
+        assertThat(result.get("type"), equalTo("patterned_text"));
+    }
+
+    public void testAnalyzerAttributeStandard() throws IOException {
+        MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("analyzer", "standard")));
+        var fieldMapper = (PatternedTextFieldMapper) mapper.mappingLookup().getMapper("field");
+        XContentBuilder builder = JsonXContent.contentBuilder().startObject();
+        fieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        var result = (Map<?, ?>) XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2().get("field");
+        assertThat(result.size(), equalTo(2));
+        assertThat(result.get("type"), equalTo("patterned_text"));
+        assertThat(result.get("analyzer"), equalTo("standard"));
+    }
+
+    public void testAnalyzerAttributeLog() throws IOException {
+        MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("analyzer", "delimiter")));
+        var fieldMapper = (PatternedTextFieldMapper) mapper.mappingLookup().getMapper("field");
+        XContentBuilder builder = JsonXContent.contentBuilder().startObject();
+        fieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        var result = (Map<?, ?>) XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2().get("field");
+        assertThat(result.size(), equalTo(1));
+        assertThat(result.get("type"), equalTo("patterned_text"));
+        assertThat(fieldMapper.getAnalyzer(), equalTo(DelimiterAnalyzer.INSTANCE));
+    }
+
+    public void testAnalyzerAttributeIllegal() throws IOException {
+        IllegalArgumentException e = (IllegalArgumentException) expectThrows(
+            MapperParsingException.class,
+            () -> createMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("analyzer", "whitespace")))
+        ).getCause();
+        assertThat(e.getMessage(), equalTo("unsupported analyzer [whitespace] for field [field], supported analyzers are [standard, log]"));
+    }
+
     @Override
     public void testSyntheticSourceKeepArrays() {
         // This mapper does not allow arrays
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java
@@ -76,7 +76,8 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
                 "field_match_only_text": { "type": "match_only_text" },
                 "field_patterned_text": {
                     "type": "patterned_text",
-                    "index_options": "%"
+                    "index_options": "%",
+                    "analyzer": "standard"
                 }
               }
             }
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/50_analyzer.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/50_analyzer.yml

Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,8 @@ protected Collection<Class<? extends Plugin>> getPlugins() {`
`76`	`76`	`"field_match_only_text": { "type": "match_only_text" },`
`77`	`77`	`"field_patterned_text": {`
`78`	`78`	`"type": "patterned_text",`
`79`		`- "index_options": "%"`
	`79`	`+ "index_options": "%",`
	`80`	`+ "analyzer": "standard"`
`80`	`81`	`}`
`81`	`82`	`}`
`82`	`83`	`}`