Merge branch '8.19' into datastreams-settings-and-mappings-overrides-8.19

Author: joegallo

docs/changelog/127664.yaml

@@ -0,0 +1,5 @@
+pr: 127664
+summary: Exclude `semantic_text` subfields from field capabilities API
+area: "Mapping"
+type: enhancement
+issues: []

docs/changelog/128504.yaml

@@ -0,0 +1,5 @@
+pr: 128504
+summary: Add l2_norm normalization support to linear retriever
+area: Relevance
+type: enhancement
+issues: []

docs/changelog/128910.yaml

@@ -0,0 +1,5 @@
+pr: 128910
+summary: Fix `FieldAttribute` name usage in `InferNonNullAggConstraint`
+area: ES|QL
+type: bug
+issues: []

docs/reference/esql/functions/description/std_dev.asciidoc

@@ -26,6 +26,40 @@ Returns top documents from a <<search-api-knn,knn search>>, in the context of a
 * <<linear-retriever,*Linear Retriever*>>.
 Combines the top results from multiple sub-retrievers using a weighted sum of their scores. Allows to specify different
 weights for each retriever, as well as independently normalize the scores from each result set.
+
+  [discrete]
+  [[retrievers-overview-linear-retriever-parameters]]
+==== Linear Retriever Parameters
+
+`retrievers`
+:   (Required, array of objects)
+    A list of the sub-retrievers' configuration, that we will take into account and whose result sets we will merge through a weighted sum. Each configuration can have a different weight and normalization depending on the specified retriever.
+
+Each entry specifies the following parameters:
+
+`retriever`
+:   (Required, a `retriever` object)
+    Specifies the retriever for which we will compute the top documents for. The retriever will produce `rank_window_size` results, which will later be merged based on the specified `weight` and `normalizer`.
+
+`weight`
+:   (Optional, float)
+    The weight that each score of this retriever’s top docs will be multiplied with. Must be greater or equal to 0. Defaults to 1.0.
+
+`normalizer`
+:   (Optional, String)
+    Specifies how we will normalize the retriever’s scores, before applying the specified `weight`. Available values are: `minmax`, `l2_norm`, and `none`. Defaults to `none`.
+
+    * `none`
+    * `minmax` : A `MinMaxScoreNormalizer` that normalizes scores based on the following formula
+
+        ```
+        score = (score - min) / (max - min)
+        ```
+
+    * `l2_norm` : An `L2ScoreNormalizer` that normalizes scores using the L2 norm of the score values.
+
+See also the hybrid search example for how to independently configure and apply normalizers to retrievers.
+
 * <<rrf-retriever,*RRF Retriever*>>.
 Combines and ranks multiple first-stage retrievers using the reciprocal rank fusion (RRF) algorithm.
 Allows you to combine multiple result sets with different relevance indicators into a single result set.

docs/reference/esql/functions/examples/std_dev.asciidoc

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
+import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
@@ -30,6 +31,7 @@
 import org.elasticsearch.tasks.CancellableTask;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -256,6 +258,14 @@ private static Predicate<MappedFieldType> buildFilter(String[] filters, String[]
             Set<String> acceptedTypes = Set.of(fieldTypes);
             fcf = ft -> acceptedTypes.contains(ft.familyTypeName());
         }
+
+        // Exclude internal ".inference" subfields of semantic_text fields from the field capabilities response
+        Collection<InferenceFieldMetadata> inferenceFields = context.getMappingLookup().inferenceFields().values();
+        for (InferenceFieldMetadata inferenceField : inferenceFields) {
+            Predicate<MappedFieldType> next = ft -> ft.name().startsWith(inferenceField.getName() + ".inference") == false;
+            fcf = fcf == null ? next : fcf.and(next);
+        }
+
         for (String filter : filters) {
             if ("parent".equals(filter) || "-parent".equals(filter)) {
                 continue;

docs/reference/esql/functions/kibana/definition/std_dev.json

@@ -473,6 +473,7 @@ static RoleDescriptor kibanaSystem(String name) {
                         "logs-tenable_sc.vulnerability-*",
                         "logs-tenable_io.vulnerability-*",
                         "logs-rapid7_insightvm.vulnerability-*",
+                        "logs-rapid7_insightvm.asset_vulnerability-*",
                         "logs-carbon_black_cloud.asset_vulnerability_summary-*"
                     )
                     .privileges("read", "view_index_metadata")