Add blockloader to patterned_text to support ESQL (#134093)

The existing blockloader for patterned_text does not work correctly, as it expects BytesRefs which were created by  BinaryFieldMapper.CustomBinaryDocValuesField. We need a new simple blockloader reader which directly maps the BytesRefs from a binary doc values to a blockloader reader. This change makes patterned_text work in ESQL.

Author: parkertimmins

server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java

@@ -864,10 +864,10 @@ public String toString() {
         }
     }
 
-    public static class BytesRefsFromBinaryBlockLoader extends DocValuesBlockLoader {
+    public static class BytesRefsFromCustomBinaryBlockLoader extends DocValuesBlockLoader {
         private final String fieldName;
 
-        public BytesRefsFromBinaryBlockLoader(String fieldName) {
+        public BytesRefsFromCustomBinaryBlockLoader(String fieldName) {
             this.fieldName = fieldName;
         }
 
@@ -882,16 +882,14 @@ public AllReader reader(LeafReaderContext context) throws IOException {
             if (docValues == null) {
                 return new ConstantNullsReader();
             }
-            return new BytesRefsFromBinary(docValues);
+            return new BytesRefsFromCustomBinary(docValues);
         }
     }
 
-    private static class BytesRefsFromBinary extends BlockDocValuesReader {
-        private final BinaryDocValues docValues;
-        private final ByteArrayStreamInput in = new ByteArrayStreamInput();
-        private final BytesRef scratch = new BytesRef();
+    abstract static class AbstractBytesRefsFromBinary extends BlockDocValuesReader {
+        protected final BinaryDocValues docValues;
 
-        BytesRefsFromBinary(BinaryDocValues docValues) {
+        AbstractBytesRefsFromBinary(BinaryDocValues docValues) {
             this.docValues = docValues;
         }
 
@@ -911,7 +909,27 @@ public void read(int docId, BlockLoader.StoredFields storedFields, Builder build
             read(docId, (BytesRefBuilder) builder);
         }
 
-        private void read(int doc, BytesRefBuilder builder) throws IOException {
+        @Override
+        public int docId() {
+            return docValues.docID();
+        }
+
+        abstract void read(int docId, BytesRefBuilder builder) throws IOException;
+    }
+
+    /**
+     * Read BinaryDocValues encoded by {@link BinaryFieldMapper.CustomBinaryDocValuesField}
+     */
+    static class BytesRefsFromCustomBinary extends AbstractBytesRefsFromBinary {
+        private final ByteArrayStreamInput in = new ByteArrayStreamInput();
+        private final BytesRef scratch = new BytesRef();
+
+        BytesRefsFromCustomBinary(BinaryDocValues docValues) {
+            super(docValues);
+        }
+
+        @Override
+        void read(int doc, BytesRefBuilder builder) throws IOException {
             if (false == docValues.advanceExact(doc)) {
                 builder.appendNull();
                 return;
@@ -939,8 +957,28 @@ private void read(int doc, BytesRefBuilder builder) throws IOException {
         }
 
         @Override
-        public int docId() {
-            return docValues.docID();
+        public String toString() {
+            return "BlockDocValuesReader.BytesCustom";
+        }
+    }
+
+    /**
+     * Read BinaryDocValues with no additional structure in the BytesRefs.
+     * Each BytesRef from the doc values maps directly to a value in the block loader.
+     */
+    public static class BytesRefsFromBinary extends AbstractBytesRefsFromBinary {
+        public BytesRefsFromBinary(BinaryDocValues docValues) {
+            super(docValues);
+        }
+
+        @Override
+        void read(int doc, BytesRefBuilder builder) throws IOException {
+            if (false == docValues.advanceExact(doc)) {
+                builder.appendNull();
+                return;
+            }
+            BytesRef bytes = docValues.binaryValue();
+            builder.appendBytesRef(bytes);
         }
 
         @Override

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBlockLoader.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb.patternedtext;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.mapper.BlockDocValuesReader;
+
+import java.io.IOException;
+
+public class PatternedTextBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
+
+    private final String templateFieldName;
+    private final String argsFieldName;
+    private final String argsInfoFieldName;
+
+    PatternedTextBlockLoader(String templateFieldName, String argsFieldName, String argsInfoFieldName) {
+        this.templateFieldName = templateFieldName;
+        this.argsFieldName = argsFieldName;
+        this.argsInfoFieldName = argsInfoFieldName;
+    }
+
+    @Override
+    public BytesRefBuilder builder(BlockFactory factory, int expectedCount) {
+        return factory.bytesRefs(expectedCount);
+    }
+
+    @Override
+    public AllReader reader(LeafReaderContext context) throws IOException {
+        var docValues = PatternedTextDocValues.from(context.reader(), templateFieldName, argsFieldName, argsInfoFieldName);
+        if (docValues == null) {
+            return new ConstantNullsReader();
+        }
+        return new BlockDocValuesReader.BytesRefsFromBinary(docValues);
+    }
+}

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java

@@ -30,7 +30,6 @@
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData;
-import org.elasticsearch.index.mapper.BlockDocValuesReader;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.StringFieldType;
@@ -252,7 +251,7 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions,
 
     @Override
     public BlockLoader blockLoader(BlockLoaderContext blContext) {
-        return new BlockDocValuesReader.BytesRefsFromBinaryBlockLoader(name());
+        return new PatternedTextBlockLoader(templateFieldName(), argsFieldName(), argsInfoFieldName());
     }
 
     @Override

x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/40_esql.yml

@@ -0,0 +1,108 @@
+---
+setup:
+  - requires:
+      cluster_features: [ "mapper.patterned_text" ]
+      reason: "patterned_text mappings are used in this test"
+
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              message:
+                type: patterned_text
+                # defaults to index_options: docs
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - { "index": { "_id": "1" } }
+          - { "@timestamp": "2025-07-17T00:00:01Z" }
+          - { "index": { "_id": "2" } }
+          - { "message": "Found 5 errors for service [cheddar1]", "@timestamp": "2025-07-17T00:00:02Z" }
+          # template_id: mOVsnxlxdac
+          - { "index": { "_id": "3"  } }
+          - { "message": "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]", "@timestamp": "2025-07-17T00:00:03Z"  }
+          # template_id: 1l_PtCLQ5xY
+          - { "index": { "_id": "4" } }
+          - { "message": "Found some errors for cheddar data service", "@timestamp": "2025-07-17T00:00:04Z"}
+          # template_id: k-2qtjujOCw
+          - { "index": { "_id": "5" } }
+          - { "message": "Found 123 errors for service [gorgonzola-24]",  "@timestamp": "2025-07-17T00:00:05Z" }
+          # template_id: mOVsnxlxdac
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: test
+
+---
+"Simple from":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | SORT @timestamp | KEEP message, message.template_id | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+  - match: {columns.1.name: "message.template_id"}
+  - match: {columns.1.type: "keyword"}
+
+  - length: {values: 5}
+  - match: {values.0.0: null }
+  - match: {values.0.1: null }
+  - match: {values.1.0: "Found 5 errors for service [cheddar1]" }
+  - match: {values.1.1: "mOVsnxlxdac" }
+  - match: {values.2.0: "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]" }
+  - match: {values.2.1: "1l_PtCLQ5xY" }
+  - match: {values.3.0: "Found some errors for cheddar data service" }
+  - match: {values.3.1: "k-2qtjujOCw" }
+  - match: {values.4.0: "Found 123 errors for service [gorgonzola-24]" }
+  - match: {values.4.1: "mOVsnxlxdac" }
+
+---
+"match query":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | WHERE MATCH(message, "gorgonzola-24") | KEEP message | LIMIT 10'
+
+  - length: {values: 1}
+  - match: {values.0.0: "Found 123 errors for service [gorgonzola-24]" }
+
+---
+"match phrase query":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | WHERE MATCH_PHRASE(message, "123 errors") | SORT @timestamp | KEEP message | LIMIT 10'
+
+  - length: {values: 2}
+  - match: {values.0.0: "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]" }
+  - match: {values.1.0: "Found 123 errors for service [gorgonzola-24]" }
+
+---
+"template_id stats":
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | STATS count(*) BY message.template_id | SORT message.template_id | LIMIT 10'
+
+  - length: {values: 4}
+  - match: {values.0.0: 1 }
+  - match: {values.0.1: "1l_PtCLQ5xY" }
+  - match: {values.1.0: 1 }
+  - match: {values.1.1: "k-2qtjujOCw" }
+  - match: {values.2.0: 2 }
+  - match: {values.2.1: "mOVsnxlxdac" }
+  - match: {values.3.0: 1 }
+  - match: {values.3.1: null }

x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java

@@ -956,7 +956,7 @@ public Query prefixQuery(
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (hasDocValues()) {
-                return new BlockDocValuesReader.BytesRefsFromBinaryBlockLoader(name());
+                return new BlockDocValuesReader.BytesRefsFromCustomBinaryBlockLoader(name());
             }
             return null;
         }