Merge branch 'main' into failures-lifecycle-config

Author: gmarouli

distribution/docker/build.gradle

@@ -590,7 +590,7 @@ for (final Architecture architecture : Architecture.values()) {
     }
     if(base == DockerBase.DEFAULT) {
       // Add additional docker hub specific context which we use solely for publishing to docker hub.
-      // At the moment it only differs in not labels added that we need for openshift certification
+      // At the moment it is exactly the same as the default context.
       addBuildDockerContextTask(architecture, base, 'DockerHubContext', "docker-hub-build-context")
     }
   }

distribution/docker/src/docker/Dockerfile.default

@@ -139,15 +139,13 @@ LABEL org.label-schema.build-date="${build_date}" \\
   org.opencontainers.image.vendor="Elastic" \\
   org.opencontainers.image.version="${version}"
 
-<% if (docker_context != 'docker-hub-build-context') { %>
 LABEL name="Elasticsearch" \\
   maintainer="[email protected]" \\
   vendor="Elastic" \\
   version="${version}" \\
   release="1" \\
   summary="Elasticsearch" \\
   description="You know, for search."
-<% } %>
 
 RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
 

docs/reference/elasticsearch/index-settings/index-modules.md

@@ -49,39 +49,27 @@ $$$index-codec$$$ `index.codec`
 
 $$$index-mode-setting$$$ `index.mode`
 :   The `index.mode` setting is used to control settings applied in specific domains like ingestion of time series data or logs. Different mutually exclusive modes exist, which are used to apply settings or default values controlling indexing of documents, sorting and other parameters whose value affects indexing or query performance.
-
-```console
-PUT my-index-000001
-{
-  "settings": {
-    "index":{
-      "mode":"standard" <1>
-    }
-  }
-}
-```
-
-1. This index uses the `standard` index mode
-
-
-Index mode supports the following values:
-
-`null`
-:   Default value (same as `standard`).
-
-`standard`
-:   Standard indexing with default settings.
-
-`lookup`
-: Index that can be used for lookup joins in ES|QL. Limited to 1 shard.
-
-
-`time_series`
-:   *(data streams only)* Index mode optimized for storage of metrics. For more information, see [Time series index settings](time-series.md).
-
-`logsdb`
-:   *(data streams only)* Index mode optimized for [logs](docs-content://manage-data/data-store/data-streams/logs-data-stream.md).
-
+        
+        **Example**
+
+      ```console
+      PUT my-index-000001
+      {
+        "settings": {
+          "index":{
+            "mode":"standard" # This index uses the `standard` index mode
+          }
+        }
+      }
+      ```
+    **Supported values**
+
+    The `index.mode` setting supports the following values:
+       - `null`:   Default value (same as `standard`).
+       -  `standard`:   Standard indexing with default settings.
+       -  `lookup`: Index that can be used for [LOOKUP JOIN](/reference/query-languages/esql/esql-lookup-join.md) in ES|QL. Limited to 1 shard.
+       - `time_series`:   *(data streams only)* Index mode optimized for storage of metrics. For more information, see [Time series index settings](time-series.md).
+       - `logsdb`: *(data streams only)* Index mode optimized for [logs](docs-content://manage-data/data-store/data-streams/logs-data-stream.md).
 
 $$$routing-partition-size$$$ `index.routing_partition_size`
 :   The number of shards a custom routing value can go to. Defaults to 1 and can only be set at index creation time. This value must be less than the `index.number_of_routing_shards` unless the `index.number_of_routing_shards` value is also 1. for more details about how this setting is used, refer to [](/reference/elasticsearch/mapping-reference/mapping-routing-field.md#routing-index-partition).

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -247,9 +247,10 @@ is not supported for querying the field data.
 
 ## Updates to `semantic_text` fields [update-script]
 
-Updates that use scripts are not supported for an index contains a
-`semantic_text` field. Even if the script targets non-`semantic_text` fields,
-the update will fail when the index contains a `semantic_text` field.
+For indices containing `semantic_text` fields, updates that use scripts have the following behavior:
+
+* Are supported through the [Update API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-update).
+* Are not supported through the [Bulk API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk-1) and will fail. Even if the script targets non-`semantic_text` fields, the update will fail when the index contains a `semantic_text` field.
 
 ## `copy_to` and multi-fields support [copy-to-support]
 
@@ -311,4 +312,4 @@ PUT test-index
   of [nested fields](/reference/elasticsearch/mapping-reference/nested.md).
 * `semantic_text` fields can’t currently be set as part
   of [Dynamic templates](docs-content://manage-data/data-store/mapping/dynamic-templates.md).
-
+* `semantic_text` fields are not supported with Cross-Cluster Search (CCS) or Cross-Cluster Replication (CCR).

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -122,21 +122,19 @@ FROM firewall_logs # The source index
 | LOOKUP JOIN threat_list ON source.ip # The lookup index and join field
 | WHERE threat_level IS NOT NULL # Filter for rows non-null threat levels
 | SORT timestamp # LOOKUP JOIN does not guarantee output order, so you must explicitly sort the results if needed
-| KEEP timestamp, source.ip, destination.ip, action, threat_level, threat_type # Keep only relevant fields
+| KEEP source.ip, action, threat_type, threat_level # Keep only relevant fields
 | LIMIT 10 # Limit the output to 10 rows
 ```
 
 ### Response
 
 A successful query will output a table. In this example, you can see that the `source.ip` field from the `firewall_logs` index is matched with the `source.ip` field in the `threat_list` index, and the corresponding `threat_level` and `threat_type` fields are added to the output.
 
-```
-   source.ip   |    action     |  threat_type  | threat_level  
----------------+---------------+---------------+---------------
-203.0.113.5    |allow          |C2_SERVER      |high           
-198.51.100.2   |block          |SCANNER        |medium         
-203.0.113.5    |allow          |C2_SERVER      |high        
-```
+|source.ip|action|threat_type|threat_level|
+|---|---|---|---|
+|203.0.113.5|allow|C2_SERVER|high|
+|198.51.100.2|block|SCANNER|medium|
+|203.0.113.5|allow|C2_SERVER|high|
 
 ### Additional examples
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -279,7 +279,10 @@ private static PolicyManager createPolicyManager() {
                     new FilesEntitlement(List.of(FileData.ofBaseDirPath(CONFIG, READ), FileData.ofBaseDirPath(DATA, READ_WRITE)))
                 )
             ),
-            new Scope("org.apache.lucene.misc", List.of(new FilesEntitlement(List.of(FileData.ofBaseDirPath(DATA, READ_WRITE))))),
+            new Scope(
+                "org.apache.lucene.misc",
+                List.of(new FilesEntitlement(List.of(FileData.ofBaseDirPath(DATA, READ_WRITE))), new ReadStoreAttributesEntitlement())
+            ),
             new Scope(
                 "org.apache.logging.log4j.core",
                 List.of(new ManageThreadsEntitlement(), new FilesEntitlement(List.of(FileData.ofBaseDirPath(LOGS, READ_WRITE))))

server/src/main/java/org/elasticsearch/cluster/project/ProjectResolver.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.FixForMultiProject;
 
 import java.util.Collection;
 import java.util.Objects;
@@ -94,6 +95,7 @@ default Collection<ProjectId> getProjectIds(ClusterState clusterState) {
     /**
      * Returns a client that executes every request in the context of the given project.
      */
+    @FixForMultiProject(description = "This recreates a client on every invocation. We should optimize this to be less wasteful")
     default Client projectClient(Client baseClient, ProjectId projectId) {
         // We only take the shortcut when the given project ID matches the "current" project ID. If it doesn't, we'll let #executeOnProject
         // take care of error handling.

server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOIndexInputSupplier.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.codec.vectors.es818;
+
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+
+import java.io.IOException;
+
+/**
+ * A hook for {@link DirectIOLucene99FlatVectorsReader} to specify the input should be opened using DirectIO.
+ * Remove when IOContext allows more extensible payloads to be specified.
+ */
+public interface DirectIOIndexInputSupplier {
+    IndexInput openInputDirect(String name, IOContext context) throws IOException;
+}

server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java

@@ -0,0 +1,72 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2024 Elasticsearch B.V.
+ */
+package org.elasticsearch.index.codec.vectors.es818;
+
+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
+import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsWriter;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+import java.io.IOException;
+
+/**
+ * Copied from Lucene99FlatVectorsFormat in Lucene 10.1
+ *
+ * This is copied to change the implementation of {@link #fieldsReader} only.
+ * The codec format itself is not changed, so we keep the original {@link #NAME}
+ */
+public class DirectIOLucene99FlatVectorsFormat extends FlatVectorsFormat {
+
+    static final String NAME = "Lucene99FlatVectorsFormat";
+    static final String META_CODEC_NAME = "Lucene99FlatVectorsFormatMeta";
+    static final String VECTOR_DATA_CODEC_NAME = "Lucene99FlatVectorsFormatData";
+    static final String META_EXTENSION = "vemf";
+    static final String VECTOR_DATA_EXTENSION = "vec";
+
+    public static final int VERSION_START = 0;
+    public static final int VERSION_CURRENT = VERSION_START;
+
+    static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16;
+    private final FlatVectorsScorer vectorsScorer;
+
+    /** Constructs a format */
+    public DirectIOLucene99FlatVectorsFormat(FlatVectorsScorer vectorsScorer) {
+        super(NAME);
+        this.vectorsScorer = vectorsScorer;
+    }
+
+    @Override
+    public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+        return new Lucene99FlatVectorsWriter(state, vectorsScorer);
+    }
+
+    @Override
+    public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+        return new DirectIOLucene99FlatVectorsReader(state, vectorsScorer);
+    }
+
+    @Override
+    public String toString() {
+        return "ES818FlatVectorsFormat(" + "vectorsScorer=" + vectorsScorer + ')';
+    }
+}