Merge branch 'main' into threadpool-merge-scheduler-sort-all-merges-take-2

Author: albertzaharovits

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -110,6 +110,8 @@ private List<File> resolveProjectLogs(File projectDir) {
         projectDirFiles.include("**/build/testrun/*/temp/**");
         projectDirFiles.include("**/build/**/hs_err_pid*.log");
         projectDirFiles.include("**/build/**/replay_pid*.log");
+        // core dump files are in the working directory of the installation, which is not project specific
+        projectDirFiles.include("distribution/**/build/install/*/core.*");
         projectDirFiles.exclude("**/build/testclusters/**/data/**");
         projectDirFiles.exclude("**/build/testclusters/**/distro/**");
         projectDirFiles.exclude("**/build/testclusters/**/repo/**");

docs/changelog/121392.yaml

@@ -0,0 +1,5 @@
+pr: 121392
+summary: Include data streams when converting an existing resource to a system resource
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/122390.yaml

@@ -0,0 +1,5 @@
+pr: 122390
+summary: Add health indicator impact to `HealthPeriodicLogger`
+area: Health
+type: enhancement
+issues: []

docs/changelog/123155.yaml

@@ -0,0 +1,5 @@
+pr: 123155
+summary: Add `ElasticInferenceServiceCompletionServiceSettings`
+area: Machine Learning
+type: bug
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -49,8 +49,24 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
         readPaths.sort(String::compareTo);
         writePaths.sort(String::compareTo);
 
-        this.readPaths = readPaths.toArray(new String[0]);
-        this.writePaths = writePaths.toArray(new String[0]);
+        this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]);
+        this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]);
+    }
+
+    private static List<String> pruneSortedPaths(List<String> paths) {
+        List<String> prunedReadPaths = new ArrayList<>();
+        if (paths.isEmpty() == false) {
+            String currentPath = paths.get(0);
+            prunedReadPaths.add(currentPath);
+            for (int i = 1; i < paths.size(); ++i) {
+                String nextPath = paths.get(i);
+                if (nextPath.startsWith(currentPath) == false) {
+                    prunedReadPaths.add(nextPath);
+                    currentPath = nextPath;
+                }
+            }
+        }
+        return prunedReadPaths;
     }
 
     public static FileAccessTree of(FilesEntitlement filesEntitlement, PathLookup pathLookup) {

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java

@@ -96,6 +96,27 @@ public void testReadWriteUnderRead() {
         assertThat(tree.canWrite(path("foo/bar")), is(true));
     }
 
+    public void testPrunedPaths() {
+        var tree = accessTree(entitlement("foo", "read", "foo/baz", "read", "foo/bar", "read"));
+        assertThat(tree.canRead(path("foo")), is(true));
+        assertThat(tree.canWrite(path("foo")), is(false));
+        assertThat(tree.canRead(path("foo/bar")), is(true));
+        assertThat(tree.canWrite(path("foo/bar")), is(false));
+        assertThat(tree.canRead(path("foo/baz")), is(true));
+        assertThat(tree.canWrite(path("foo/baz")), is(false));
+        // also test a non-existent subpath
+        assertThat(tree.canRead(path("foo/barf")), is(true));
+        assertThat(tree.canWrite(path("foo/barf")), is(false));
+
+        tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write"));
+        assertThat(tree.canRead(path("foo")), is(true));
+        assertThat(tree.canWrite(path("foo")), is(false));
+        assertThat(tree.canRead(path("foo/bar")), is(true));
+        assertThat(tree.canWrite(path("foo/bar")), is(true));
+        assertThat(tree.canRead(path("foo/baz")), is(true));
+        assertThat(tree.canWrite(path("foo/baz")), is(false));
+    }
+
     public void testReadWithRelativePath() {
         for (var dir : List.of("config", "home")) {
             var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read", "relative_to", dir)));

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java

@@ -31,15 +31,13 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
-import org.hamcrest.Matcher;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 
@@ -48,7 +46,6 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notANumber;
 
 public class ScaledFloatFieldMapperTests extends NumberFieldMapperTests {
 
@@ -382,7 +379,7 @@ public SyntheticSourceExample example(int maxValues) {
             if (randomBoolean()) {
                 Value v = generateValue();
                 if (v.malformedOutput == null) {
-                    return new SyntheticSourceExample(v.input, v.output, roundDocValues(v.output), this::mapping);
+                    return new SyntheticSourceExample(v.input, v.output, this::mapping);
                 }
                 return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping);
             }
@@ -396,9 +393,7 @@ public SyntheticSourceExample example(int maxValues) {
             List<Object> outList = Stream.concat(outputFromDocValues.stream(), malformedOutput).toList();
             Object out = outList.size() == 1 ? outList.get(0) : outList;
 
-            List<Double> outBlockList = outputFromDocValues.stream().map(this::roundDocValues).sorted().toList();
-            Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
-            return new SyntheticSourceExample(in, out, outBlock, this::mapping);
+            return new SyntheticSourceExample(in, out, this::mapping);
         }
 
         private record Value(Object input, Double output, Object malformedOutput) {}
@@ -442,16 +437,6 @@ private double round(double d) {
             return decoded;
         }
 
-        private double roundDocValues(double d) {
-            // Special case due to rounding, see implementation.
-            if (Math.abs(d) == Double.MAX_VALUE) {
-                return d;
-            }
-
-            long encoded = Math.round(d * scalingFactor);
-            return encoded * (1 / scalingFactor);
-        }
-
         private void mapping(XContentBuilder b) throws IOException {
             b.field("type", "scaled_float");
             b.field("scaling_factor", scalingFactor);
@@ -475,14 +460,9 @@ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
         }
     }
 
-    @Override
-    protected Function<Object, Object> loadBlockExpected() {
-        return v -> (Number) v;
-    }
-
-    @Override
-    protected Matcher<?> blockItemMatcher(Object expected) {
-        return "NaN".equals(expected) ? notANumber() : equalTo(expected);
+    protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
+        assumeTrue("Disabled, tested by ScaledFloatFieldBlockLoaderTests instead", false);
+        return null;
     }
 
     @Override

muted-tests.yml

@@ -332,15 +332,21 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithPartialResultsIT
   method: testPartialResults
   issue: https://github.com/elastic/elasticsearch/issues/123101
-- class: org.elasticsearch.index.mapper.extras.ScaledFloatFieldMapperTests
-  method: testBlockLoaderFromRowStrideReader
-  issue: https://github.com/elastic/elasticsearch/issues/123126
-- class: org.elasticsearch.index.mapper.extras.ScaledFloatFieldMapperTests
-  method: testBlockLoaderFromRowStrideReaderWithSyntheticSource
-  issue: https://github.com/elastic/elasticsearch/issues/123145
 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
   method: testStopQueryLocal
   issue: https://github.com/elastic/elasticsearch/issues/121672
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/snapshot-restore/restore-snapshot/line_408}
+  issue: https://github.com/elastic/elasticsearch/issues/123192
+- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
+  method: testRestoredIndexManagedByLocalPolicySkipsIllegalActions
+  issue: https://github.com/elastic/elasticsearch/issues/123202
+- class: org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT
+  method: testHistoryIsWrittenWithFailure
+  issue: https://github.com/elastic/elasticsearch/issues/123203
+- class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT
+  method: testSearchableSnapshotAction
+  issue: https://github.com/elastic/elasticsearch/issues/123214
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -187,6 +187,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion REMOVE_DESIRED_NODE_VERSION_90 = def(9_000_0_03);
     public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION_90 = def(9_000_0_04);
     public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_9_0 = def(9_000_0_05);
+    public static final TransportVersion BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_90 = def(9_000_0_06);
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
     public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00);
     public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED = def(9_003_0_00);
@@ -201,6 +202,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion REMOVE_REPOSITORY_CONFLICT_MESSAGE = def(9_012_0_00);
     public static final TransportVersion RERANKER_FAILURES_ALLOWED = def(9_013_0_00);
     public static final TransportVersion VOYAGE_AI_INTEGRATION_ADDED = def(9_014_0_00);
+    public static final TransportVersion BYTE_SIZE_VALUE_ALWAYS_USES_BYTES = def(9_015_0_00);
+
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -301,6 +301,13 @@ public boolean isFailureStoreIndex(String indexName) {
         return failureIndices.containsIndex(indexName);
     }
 
+    /**
+     * Returns true if the index name provided belongs to this data stream.
+     */
+    public boolean containsIndex(String indexName) {
+        return backingIndices.containsIndex(indexName) || failureIndices.containsIndex(indexName);
+    }
+
     public DataStreamOptions getDataStreamOptions() {
         return dataStreamOptions;
     }
@@ -782,8 +789,9 @@ public DataStream addBackingIndex(Metadata clusterMetadata, Index index) {
         // ensure that no aliases reference index
         ensureNoAliasesOnIndex(clusterMetadata, index);
 
-        List<Index> backingIndices = new ArrayList<>(this.backingIndices.indices);
-        backingIndices.add(0, index);
+        List<Index> backingIndices = new ArrayList<>(this.backingIndices.indices.size() + 1);
+        backingIndices.add(index);
+        backingIndices.addAll(this.backingIndices.indices);
         assert backingIndices.size() == this.backingIndices.indices.size() + 1;
         return copy().setBackingIndices(this.backingIndices.copy().setIndices(backingIndices).build())
             .setGeneration(generation + 1)
@@ -808,8 +816,9 @@ public DataStream addFailureStoreIndex(Metadata clusterMetadata, Index index) {
 
         ensureNoAliasesOnIndex(clusterMetadata, index);
 
-        List<Index> updatedFailureIndices = new ArrayList<>(failureIndices.indices);
-        updatedFailureIndices.add(0, index);
+        List<Index> updatedFailureIndices = new ArrayList<>(failureIndices.indices.size() + 1);
+        updatedFailureIndices.add(index);
+        updatedFailureIndices.addAll(failureIndices.indices);
         assert updatedFailureIndices.size() == failureIndices.indices.size() + 1;
         return copy().setFailureIndices(failureIndices.copy().setIndices(updatedFailureIndices).build())
             .setGeneration(generation + 1)
@@ -1039,7 +1048,7 @@ private boolean isIndexOlderThan(
      * we return false.
      */
     public boolean isIndexManagedByDataStreamLifecycle(Index index, Function<String, IndexMetadata> indexMetadataSupplier) {
-        if (backingIndices.containsIndex(index.getName()) == false && failureIndices.containsIndex(index.getName()) == false) {
+        if (containsIndex(index.getName()) == false) {
             return false;
         }
         IndexMetadata indexMetadata = indexMetadataSupplier.apply(index.getName());