diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index f2324d70ce750..6061682d7146a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -864,10 +864,10 @@ public String toString() { } } - public static class BytesRefsFromBinaryBlockLoader extends DocValuesBlockLoader { + public static class BytesRefsFromCustomBinaryBlockLoader extends DocValuesBlockLoader { private final String fieldName; - public BytesRefsFromBinaryBlockLoader(String fieldName) { + public BytesRefsFromCustomBinaryBlockLoader(String fieldName) { this.fieldName = fieldName; } @@ -882,16 +882,14 @@ public AllReader reader(LeafReaderContext context) throws IOException { if (docValues == null) { return new ConstantNullsReader(); } - return new BytesRefsFromBinary(docValues); + return new BytesRefsFromCustomBinary(docValues); } } - private static class BytesRefsFromBinary extends BlockDocValuesReader { - private final BinaryDocValues docValues; - private final ByteArrayStreamInput in = new ByteArrayStreamInput(); - private final BytesRef scratch = new BytesRef(); + abstract static class AbstractBytesRefsFromBinary extends BlockDocValuesReader { + protected final BinaryDocValues docValues; - BytesRefsFromBinary(BinaryDocValues docValues) { + AbstractBytesRefsFromBinary(BinaryDocValues docValues) { this.docValues = docValues; } @@ -911,7 +909,27 @@ public void read(int docId, BlockLoader.StoredFields storedFields, Builder build read(docId, (BytesRefBuilder) builder); } - private void read(int doc, BytesRefBuilder builder) throws IOException { + @Override + public int docId() { + return docValues.docID(); + } + + abstract void read(int docId, BytesRefBuilder builder) throws IOException; + } + + /** + * Read BinaryDocValues encoded by {@link BinaryFieldMapper.CustomBinaryDocValuesField} + */ + static class BytesRefsFromCustomBinary extends AbstractBytesRefsFromBinary { + private final ByteArrayStreamInput in = new ByteArrayStreamInput(); + private final BytesRef scratch = new BytesRef(); + + BytesRefsFromCustomBinary(BinaryDocValues docValues) { + super(docValues); + } + + @Override + void read(int doc, BytesRefBuilder builder) throws IOException { if (false == docValues.advanceExact(doc)) { builder.appendNull(); return; @@ -939,8 +957,28 @@ private void read(int doc, BytesRefBuilder builder) throws IOException { } @Override - public int docId() { - return docValues.docID(); + public String toString() { + return "BlockDocValuesReader.BytesCustom"; + } + } + + /** + * Read BinaryDocValues with no additional structure in the BytesRefs. + * Each BytesRef from the doc values maps directly to a value in the block loader. + */ + public static class BytesRefsFromBinary extends AbstractBytesRefsFromBinary { + public BytesRefsFromBinary(BinaryDocValues docValues) { + super(docValues); + } + + @Override + void read(int doc, BytesRefBuilder builder) throws IOException { + if (false == docValues.advanceExact(doc)) { + builder.appendNull(); + return; + } + BytesRef bytes = docValues.binaryValue(); + builder.appendBytesRef(bytes); } @Override diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java new file mode 100644 index 0000000000000..a4ca13520d7e8 --- /dev/null +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb.patternedtext; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.mapper.BlockDocValuesReader; + +import java.io.IOException; + +public class PatternedTextBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { + + private final String templateFieldName; + private final String argsFieldName; + private final String argsInfoFieldName; + + PatternedTextBlockLoader(String templateFieldName, String argsFieldName, String argsInfoFieldName) { + this.templateFieldName = templateFieldName; + this.argsFieldName = argsFieldName; + this.argsInfoFieldName = argsInfoFieldName; + } + + @Override + public BytesRefBuilder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + var docValues = PatternedTextDocValues.from(context.reader(), templateFieldName, argsFieldName, argsInfoFieldName); + if (docValues == null) { + return new ConstantNullsReader(); + } + return new BlockDocValuesReader.BytesRefsFromBinary(docValues); + } +} diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java index 2c7f9b59ee4dc..e195e4f744ded 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; -import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; @@ -252,7 +251,7 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return new BlockDocValuesReader.BytesRefsFromBinaryBlockLoader(name()); + return new PatternedTextBlockLoader(templateFieldName(), argsFieldName(), argsInfoFieldName()); } @Override diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/40_esql.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/40_esql.yml new file mode 100644 index 0000000000000..b04691290e1a4 --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/40_esql.yml @@ -0,0 +1,108 @@ +--- +setup: + - requires: + cluster_features: [ "mapper.patterned_text" ] + reason: "patterned_text mappings are used in this test" + + - do: + indices.create: + index: test + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + "@timestamp": + type: date + message: + type: patterned_text + # defaults to index_options: docs + + - do: + bulk: + index: test + refresh: true + body: + - { "index": { "_id": "1" } } + - { "@timestamp": "2025-07-17T00:00:01Z" } + - { "index": { "_id": "2" } } + - { "message": "Found 5 errors for service [cheddar1]", "@timestamp": "2025-07-17T00:00:02Z" } + # template_id: mOVsnxlxdac + - { "index": { "_id": "3" } } + - { "message": "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]", "@timestamp": "2025-07-17T00:00:03Z" } + # template_id: 1l_PtCLQ5xY + - { "index": { "_id": "4" } } + - { "message": "Found some errors for cheddar data service", "@timestamp": "2025-07-17T00:00:04Z"} + # template_id: k-2qtjujOCw + - { "index": { "_id": "5" } } + - { "message": "Found 123 errors for service [gorgonzola-24]", "@timestamp": "2025-07-17T00:00:05Z" } + # template_id: mOVsnxlxdac + +--- +teardown: + - do: + indices.delete: + index: test + +--- +"Simple from": + - do: + esql.query: + body: + query: 'FROM test | SORT @timestamp | KEEP message, message.template_id | LIMIT 10' + + - match: {columns.0.name: "message"} + - match: {columns.0.type: "text"} + - match: {columns.1.name: "message.template_id"} + - match: {columns.1.type: "keyword"} + + - length: {values: 5} + - match: {values.0.0: null } + - match: {values.0.1: null } + - match: {values.1.0: "Found 5 errors for service [cheddar1]" } + - match: {values.1.1: "mOVsnxlxdac" } + - match: {values.2.0: "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]" } + - match: {values.2.1: "1l_PtCLQ5xY" } + - match: {values.3.0: "Found some errors for cheddar data service" } + - match: {values.3.1: "k-2qtjujOCw" } + - match: {values.4.0: "Found 123 errors for service [gorgonzola-24]" } + - match: {values.4.1: "mOVsnxlxdac" } + +--- +"match query": + - do: + esql.query: + body: + query: 'FROM test | WHERE MATCH(message, "gorgonzola-24") | KEEP message | LIMIT 10' + + - length: {values: 1} + - match: {values.0.0: "Found 123 errors for service [gorgonzola-24]" } + +--- +"match phrase query": + - do: + esql.query: + body: + query: 'FROM test | WHERE MATCH_PHRASE(message, "123 errors") | SORT @timestamp | KEEP message | LIMIT 10' + + - length: {values: 2} + - match: {values.0.0: "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]" } + - match: {values.1.0: "Found 123 errors for service [gorgonzola-24]" } + +--- +"template_id stats": + - do: + esql.query: + body: + query: 'FROM test | STATS count(*) BY message.template_id | SORT message.template_id | LIMIT 10' + + - length: {values: 4} + - match: {values.0.0: 1 } + - match: {values.0.1: "1l_PtCLQ5xY" } + - match: {values.1.0: 1 } + - match: {values.1.1: "k-2qtjujOCw" } + - match: {values.2.0: 2 } + - match: {values.2.1: "mOVsnxlxdac" } + - match: {values.3.0: 1 } + - match: {values.3.1: null } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 3aa43fa925747..ff89074c3f063 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -956,7 +956,7 @@ public Query prefixQuery( @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return new BlockDocValuesReader.BytesRefsFromBinaryBlockLoader(name()); + return new BlockDocValuesReader.BytesRefsFromCustomBinaryBlockLoader(name()); } return null; }