Merge branch 'main' into get-desired-balance-node-weights

Author: joshua-adams-1

docs/changelog/131531.yaml

@@ -0,0 +1,6 @@
+pr: 131531
+summary: Integrate LIKE/RLIKE LIST with `ReplaceStringCasingWithInsensitiveRegexMatch`
+  rule
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/query-languages/esql/_snippets/commands/layout/completion.md

@@ -0,0 +1,106 @@
+## `COMPLETION` [esql-completion]
+
+```yaml {applies_to}
+serverless: preview
+stack: preview 9.1.0
+```
+
+The `COMPLETION` command allows you to send prompts and context to a Large Language Model (LLM) directly within your ES|QL queries, to perform text generation tasks.
+
+**Syntax**
+
+```esql
+COMPLETION [column =] prompt WITH inference_id
+```
+
+**Parameters**
+
+`column`
+:   (Optional) The name of the output column containing the LLM's response.
+    If not specified, the results will be stored in a column named `completion`.
+    If the specified column already exists, it will be overwritten with the new results.
+
+`prompt`
+:   The input text or expression used to prompt the LLM.
+    This can be a string literal or a reference to a column containing text.
+
+`inference_id`
+:   The ID of the [inference endpoint](docs-content://explore-analyze/elastic-inference/inference-api.md) to use for the task.
+    The inference endpoint must be configured with the `completion` task type.
+
+**Description**
+
+The `COMPLETION` command provides a general-purpose interface for
+text generation tasks using a Large Language Model (LLM) in ES|QL.
+
+`COMPLETION` supports a wide range of text generation tasks. Depending on your
+prompt and the model you use, you can perform arbitrary text generation tasks
+including:
+
+- Question answering
+- Summarization
+- Translation
+- Content rewriting
+- Creative generation
+
+**Requirements**
+
+To use this command, you must deploy your LLM model in Elasticsearch as
+an [≈inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the
+task type `completion`.
+
+**Examples**
+
+Use the default column name (results stored in `completion` column):
+
+```esql
+ROW question = "What is Elasticsearch?"
+| COMPLETION question WITH test_completion_model
+| KEEP question, completion
+```
+
+| question:keyword       | completion:keyword                        |
+|------------------------|-------------------------------------------|
+| What is Elasticsearch? | A distributed search and analytics engine |
+
+Specify the output column (results stored in `answer` column):
+
+```esql
+ROW question = "What is Elasticsearch?"
+| COMPLETION answer = question WITH test_completion_model
+| KEEP question, answer
+```
+
+| question:keyword | answer:keyword |
+| --- | --- |
+| What is Elasticsearch? | A distributed search and analytics engine |
+
+Summarize the top 10 highest-rated movies using a prompt:
+
+```esql
+FROM movies
+| SORT rating DESC
+| LIMIT 10
+| EVAL prompt = CONCAT(
+   "Summarize this movie using the following information: \n",
+   "Title: ", title, "\n",
+   "Synopsis: ", synopsis, "\n",
+   "Actors: ", MV_CONCAT(actors, ", "), "\n",
+  )
+| COMPLETION summary = prompt WITH test_completion_model
+| KEEP title, summary, rating
+```
+
+
+| title:keyword | summary:keyword | rating:double |
+| --- | --- | --- |
+| The Shawshank Redemption | A tale of hope and redemption in prison. | 9.3 |
+| The Godfather | A mafia family's rise and fall. | 9.2 |
+| The Dark Knight | Batman battles the Joker in Gotham. | 9.0 |
+| Pulp Fiction | Interconnected crime stories with dark humor. | 8.9 |
+| Fight Club | A man starts an underground fight club. | 8.8 |
+| Inception | A thief steals secrets through dreams. | 8.8 |
+| The Matrix | A hacker discovers reality is a simulation. | 8.7 |
+| Parasite | Class conflict between two families. | 8.6 |
+| Interstellar | A team explores space to save humanity. | 8.6 |
+| The Prestige | Rival magicians engage in dangerous competition. | 8.5 |

docs/reference/query-languages/esql/_snippets/lists/processing-commands.md

@@ -1,4 +1,5 @@
 * [preview] [`CHANGE_POINT`](../../commands/processing-commands.md#esql-change_point)
+* [preview] [`COMPLETION`](../../commands/processing-commands.md#esql-completion)
 * [`DISSECT`](../../commands/processing-commands.md#esql-dissect)
 * [`DROP`](../../commands/processing-commands.md#esql-drop)
 * [`ENRICH`](../../commands/processing-commands.md#esql-enrich)

docs/reference/query-languages/esql/commands/processing-commands.md

@@ -20,6 +20,9 @@ mapped_pages:
 :::{include} ../_snippets/commands/layout/change_point.md
 :::
 
+:::{include} ../_snippets/commands/layout/completion.md
+:::
+
 :::{include} ../_snippets/commands/layout/dissect.md
 :::
 

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -347,7 +347,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_LLAMA_ADDED = def(9_125_0_00);
     public static final TransportVersion SHARD_WRITE_LOAD_IN_CLUSTER_INFO = def(9_126_0_00);
     public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS = def(9_127_0_00);
-    public static final TransportVersion NODE_WEIGHTS_ADDED_TO_NODE_BALANCE_STATS = def(9_128_0_00);
+    public static final TransportVersion ESQL_TOPN_TIMINGS = def(9_128_0_00);
+    public static final TransportVersion NODE_WEIGHTS_ADDED_TO_NODE_BALANCE_STATS = def(9_129_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -209,7 +209,8 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
         private final IndexAnalyzers indexAnalyzers;
         private final ScriptCompiler scriptCompiler;
         private final IndexVersion indexCreatedVersion;
-        private final boolean useDocValuesSkipper;
+        private final boolean enableDocValuesSkipper;
+        private final boolean forceDocValuesSkipper;
         private final SourceKeepMode indexSourceKeepMode;
 
         public Builder(final String name, final MappingParserContext mappingParserContext) {
@@ -222,6 +223,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
                 mappingParserContext.getIndexSettings().getMode(),
                 mappingParserContext.getIndexSettings().getIndexSortConfig(),
                 USE_DOC_VALUES_SKIPPER.get(mappingParserContext.getSettings()),
+                false,
                 mappingParserContext.getIndexSettings().sourceKeepMode()
             );
         }
@@ -243,6 +245,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
                 IndexMode.STANDARD,
                 null,
                 false,
+                false,
                 sourceKeepMode
             );
         }
@@ -255,7 +258,8 @@ private Builder(
             IndexVersion indexCreatedVersion,
             IndexMode indexMode,
             IndexSortConfig indexSortConfig,
-            boolean useDocValuesSkipper,
+            boolean enableDocValuesSkipper,
+            boolean forceDocValuesSkipper,
             SourceKeepMode indexSourceKeepMode
         ) {
             super(name);
@@ -293,14 +297,36 @@ private Builder(
                 });
             this.indexSortConfig = indexSortConfig;
             this.indexMode = indexMode;
-            this.useDocValuesSkipper = useDocValuesSkipper;
+            this.enableDocValuesSkipper = enableDocValuesSkipper;
+            this.forceDocValuesSkipper = forceDocValuesSkipper;
             this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         public Builder(String name, IndexVersion indexCreatedVersion) {
             this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE);
         }
 
+        public static Builder buildWithDocValuesSkipper(
+            String name,
+            IndexMode indexMode,
+            IndexVersion indexCreatedVersion,
+            boolean enableDocValuesSkipper
+        ) {
+            return new Builder(
+                name,
+                null,
+                ScriptCompiler.NONE,
+                Integer.MAX_VALUE,
+                indexCreatedVersion,
+                indexMode,
+                // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed.
+                null,
+                enableDocValuesSkipper,
+                true,
+                SourceKeepMode.NONE
+            );
+        }
+
         public Builder ignoreAbove(int ignoreAbove) {
             this.ignoreAbove.setValue(ignoreAbove);
             return this;
@@ -422,7 +448,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType
         @Override
         public KeywordFieldMapper build(MapperBuilderContext context) {
             FieldType fieldtype = resolveFieldType(
-                useDocValuesSkipper,
+                enableDocValuesSkipper,
+                forceDocValuesSkipper,
+                hasDocValues,
                 indexCreatedVersion,
                 indexSortConfig,
                 indexMode,
@@ -460,24 +488,30 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
                 buildFieldType(context, fieldtype),
                 builderParams(this, context),
                 context.isSourceSynthetic(),
-                useDocValuesSkipper,
                 this,
                 offsetsFieldName,
                 indexSourceKeepMode
             );
         }
 
-        private FieldType resolveFieldType(
-            final boolean useDocValuesSkipper,
+        private static FieldType resolveFieldType(
+            final boolean enableDocValuesSkipper,
+            final boolean forceDocValuesSkipper,
+            final Parameter<Boolean> hasDocValues,
             final IndexVersion indexCreatedVersion,
             final IndexSortConfig indexSortConfig,
             final IndexMode indexMode,
             final String fullFieldName
         ) {
-            if (useDocValuesSkipper
-                && indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX)
-                && shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) {
-                return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
+            if (enableDocValuesSkipper) {
+                if (forceDocValuesSkipper) {
+                    assert hasDocValues.getValue();
+                    return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
+                }
+                if (indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX)
+                    && shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) {
+                    return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
+                }
             }
             return new FieldType(Defaults.FIELD_TYPE);
         }
@@ -1088,7 +1122,8 @@ public String originalName() {
     private final int ignoreAboveDefault;
     private final IndexMode indexMode;
     private final IndexSortConfig indexSortConfig;
-    private final boolean useDocValuesSkipper;
+    private final boolean enableDocValuesSkipper;
+    private final boolean forceDocValuesSkipper;
     private final String offsetsFieldName;
     private final SourceKeepMode indexSourceKeepMode;
     private final String originalName;
@@ -1099,7 +1134,6 @@ private KeywordFieldMapper(
         KeywordFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean isSyntheticSource,
-        boolean useDocValuesSkipper,
         Builder builder,
         String offsetsFieldName,
         SourceKeepMode indexSourceKeepMode
@@ -1120,7 +1154,8 @@ private KeywordFieldMapper(
         this.ignoreAboveDefault = builder.ignoreAboveDefault;
         this.indexMode = builder.indexMode;
         this.indexSortConfig = builder.indexSortConfig;
-        this.useDocValuesSkipper = useDocValuesSkipper;
+        this.enableDocValuesSkipper = builder.enableDocValuesSkipper;
+        this.forceDocValuesSkipper = builder.forceDocValuesSkipper;
         this.offsetsFieldName = offsetsFieldName;
         this.indexSourceKeepMode = indexSourceKeepMode;
         this.originalName = mappedFieldType.originalName();
@@ -1219,7 +1254,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
             throw new IllegalArgumentException(msg);
         }
 
-        Field field = new KeywordField(fieldType().name(), binaryValue, fieldType);
+        Field field = buildKeywordField(binaryValue);
         context.doc().add(field);
 
         if (fieldType().hasDocValues() == false && fieldType.omitNorms()) {
@@ -1276,11 +1311,16 @@ public FieldMapper.Builder getMergeBuilder() {
             indexCreatedVersion,
             indexMode,
             indexSortConfig,
-            useDocValuesSkipper,
+            enableDocValuesSkipper,
+            forceDocValuesSkipper,
             indexSourceKeepMode
         ).dimension(fieldType().isDimension()).init(this);
     }
 
+    public Field buildKeywordField(BytesRef binaryValue) {
+        return new KeywordField(fieldType().name(), binaryValue, fieldType);
+    }
+
     @Override
     public void doValidate(MappingLookup lookup) {
         if (fieldType().isDimension() && null != lookup.nestedLookup().getNestedParent(fullPath())) {

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.util.Objects;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isStringAndExact;
@@ -67,6 +68,14 @@ public Boolean fold(FoldContext ctx) {
         throw new UnsupportedOperationException();
     }
 
+    /**
+     * Returns an equivalent optimized expression taking into account the case of the pattern(s)
+     * @param unwrappedField the field with to_upper/to_lower function removed
+     * @param matchesCaseFn a predicate to check if a pattern matches the case
+     * @return an optimized equivalent Expression or this if no optimization is possible
+     */
+    public abstract Expression optimizeStringCasingWithInsensitiveRegexMatch(Expression unwrappedField, Predicate<String> matchesCaseFn);
+
     @Override
     public boolean equals(Object obj) {
         if (super.equals(obj)) {

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNOperator.java

@@ -308,6 +308,9 @@ public String describe() {
 
     private Iterator<Page> output;
 
+    private long receiveNanos;
+    private long emitNanos;
+
     /**
      * Count of pages that have been received by this operator.
      */
@@ -387,6 +390,7 @@ public boolean needsInput() {
 
     @Override
     public void addInput(Page page) {
+        long start = System.nanoTime();
         /*
          * Since row tracks memory we have to be careful to close any unused rows,
          * including any rows that fail while constructing because they allocate
@@ -422,13 +426,16 @@ public void addInput(Page page) {
             page.releaseBlocks();
             pagesReceived++;
             rowsReceived += page.getPositionCount();
+            receiveNanos += System.nanoTime() - start;
         }
     }
 
     @Override
     public void finish() {
         if (output == null) {
+            long start = System.nanoTime();
             output = toPages();
+            emitNanos += System.nanoTime() - start;
         }
     }
 
@@ -588,7 +595,16 @@ public long ramBytesUsed() {
 
     @Override
     public Status status() {
-        return new TopNOperatorStatus(inputQueue.size(), ramBytesUsed(), pagesReceived, pagesEmitted, rowsReceived, rowsEmitted);
+        return new TopNOperatorStatus(
+            receiveNanos,
+            emitNanos,
+            inputQueue.size(),
+            ramBytesUsed(),
+            pagesReceived,
+            pagesEmitted,
+            rowsReceived,
+            rowsEmitted
+        );
     }
 
     @Override