Merge branch 'main' into lookupJoin

Author: julian-elastic

docs/changelog/130092.yaml

@@ -0,0 +1,5 @@
+pr: 130092
+summary: "Added Llama provider support to the Inference Plugin"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/131395.yaml

@@ -0,0 +1,5 @@
+pr: 131395
+summary: Enable failure store for newly created OTel data streams
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/131442.yaml

@@ -0,0 +1,5 @@
+pr: 131442
+summary: Track inference deployments
+area: Machine Learning
+type: enhancement
+issues: []

docs/reference/elasticsearch/index-settings/slow-log.md

@@ -20,6 +20,7 @@ Events that meet the specified threshold are emitted into [{{es}} logging](docs-
 * If [{{es}} monitoring](docs-content://deploy-manage/monitor/stack-monitoring.md) is enabled, from [Stack Monitoring](docs-content://deploy-manage/monitor/monitoring-data/visualizing-monitoring-data.md). Slow log events have a `logger` value of `index.search.slowlog` or `index.indexing.slowlog`.
 * From local {{es}} service logs directory. Slow log files have a suffix of `_index_search_slowlog.json` or `_index_indexing_slowlog.json`.
 
+See this [this video](https://www.youtube.com/watch?v=ulUPJshB5bU) for a walkthrough of setting and reviewing slow logs.
 
 ## Slow log format [slow-log-format]
 

libs/x-content/src/main/java/org/elasticsearch/xcontent/ConstructingObjectParser.java

@@ -220,6 +220,27 @@ public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Contex
         }
     }
 
+    /**
+     * Declare a field that is an array of objects or null. Used to avoid calling the consumer when used with
+     * {@link #optionalConstructorArg()} or {@link #constructorArg()}.
+     * @param consumer Consumer that will be passed as is to the {@link #declareField(BiConsumer, ContextParser, ParseField, ValueType)}.
+     * @param objectParser Parser that will parse the objects in the array, checking for nulls.
+     * @param field Field to declare.
+     */
+    @Override
+    public <T> void declareObjectArrayOrNull(
+        BiConsumer<Value, List<T>> consumer,
+        ContextParser<Context, T> objectParser,
+        ParseField field
+    ) {
+        declareField(
+            consumer,
+            (p, c) -> p.currentToken() == XContentParser.Token.VALUE_NULL ? null : parseArray(p, c, objectParser),
+            field,
+            ValueType.OBJECT_ARRAY_OR_NULL
+        );
+    }
+
     @Override
     public <T> void declareNamedObject(
         BiConsumer<Value, T> consumer,

muted-tests.yml

@@ -478,6 +478,12 @@ tests:
 - class: org.elasticsearch.xpack.downsample.DownsampleIT
   method: testAggMetricInEsqlTSAfterDownsampling
   issue: https://github.com/elastic/elasticsearch/issues/131500
+- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/131508
+- class: org.elasticsearch.action.admin.cluster.node.tasks.CancellableTasksIT
+  method: testRemoveBanParentsOnDisconnect
+  issue: https://github.com/elastic/elasticsearch/issues/131562
 
 # Examples:
 #

server/src/main/java/module-info.java

@@ -484,4 +484,5 @@
     exports org.elasticsearch.index.codec.perfield;
     exports org.elasticsearch.index.codec.vectors to org.elasticsearch.test.knn;
     exports org.elasticsearch.index.codec.vectors.es818 to org.elasticsearch.test.knn;
+    exports org.elasticsearch.inference.telemetry;
 }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -342,7 +342,10 @@ static TransportVersion def(int id) {
     public static final TransportVersion NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO = def(9_121_0_00);
     public static final TransportVersion ESQL_CATEGORIZE_OPTIONS = def(9_122_0_00);
     public static final TransportVersion ML_INFERENCE_AZURE_AI_STUDIO_RERANK_ADDED = def(9_123_0_00);
-    public static final TransportVersion ESQL_LOOKUP_JOIN_ON_MANY_FIELDS = def(9_124_0_00);
+    public static final TransportVersion PROJECT_STATE_REGISTRY_ENTRY = def(9_124_0_00);
+    public static final TransportVersion ML_INFERENCE_LLAMA_ADDED = def(9_125_0_00);
+    public static final TransportVersion ESQL_LOOKUP_JOIN_ON_MANY_FIELDS = def(9_126_0_00);
+
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -835,10 +835,6 @@ private Iterator<? extends ToXContent> toXContentChunkedWithSingleProjectFormat(
         );
     }
 
-    private static final DiffableUtils.KeySerializer<ProjectId> PROJECT_ID_SERIALIZER = DiffableUtils.getWriteableKeySerializer(
-        ProjectId.READER
-    );
-
     private static class MetadataDiff implements Diff<Metadata> {
 
         private final long version;
@@ -880,7 +876,7 @@ private static class MetadataDiff implements Diff<Metadata> {
                 multiProject = null;
             } else {
                 singleProject = null;
-                multiProject = DiffableUtils.diff(before.projectMetadata, after.projectMetadata, PROJECT_ID_SERIALIZER);
+                multiProject = DiffableUtils.diff(before.projectMetadata, after.projectMetadata, ProjectId.PROJECT_ID_SERIALIZER);
             }
 
             if (empty) {
@@ -1004,7 +1000,7 @@ private MetadataDiff(StreamInput in) throws IOException {
                 singleProject = null;
                 multiProject = DiffableUtils.readJdkMapDiff(
                     in,
-                    PROJECT_ID_SERIALIZER,
+                    ProjectId.PROJECT_ID_SERIALIZER,
                     ProjectMetadata::readFrom,
                     ProjectMetadata.ProjectMetadataDiff::new
                 );
@@ -1059,7 +1055,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 if (multiProject != null) {
                     multiProject.writeTo(out);
                 } else {
-                    DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER).writeTo(out);
+                    DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, ProjectId.PROJECT_ID_SERIALIZER).writeTo(out);
                 }
             }
         }

server/src/main/java/org/elasticsearch/cluster/metadata/ProjectId.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -26,6 +27,7 @@ public class ProjectId implements Writeable, ToXContent {
     private static final String DEFAULT_STRING = "default";
     public static final ProjectId DEFAULT = new ProjectId(DEFAULT_STRING);
     public static final Reader<ProjectId> READER = ProjectId::readFrom;
+    public static final DiffableUtils.KeySerializer<ProjectId> PROJECT_ID_SERIALIZER = DiffableUtils.getWriteableKeySerializer(READER);
     private static final int MAX_LENGTH = 128;
 
     private final String id;