Merge branch 'main' into svilen/121525

Author: svilen-mihaylov-elastic

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -37,8 +37,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:logstash");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:mixed-tier-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:repository-old-versions");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade");
@@ -60,12 +58,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:async-search:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:autoscaling:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:downgrade-to-basic-license");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:multi-cluster");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:non-compliant-license");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:restart");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:early-deprecation-rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:with-security");
@@ -92,8 +84,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:tls-basic");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:multi-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:rolling-upgrade");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:multi-node");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:fs");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:hdfs");
@@ -106,12 +96,10 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:mixed-node");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:with-ssl");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:without-ssl");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:stack:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:text-structure:qa:text-structure-with-security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:multi-cluster-tests-with-security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:multi-node-tests");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:single-node-tests");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile:qa:multi-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:with-security");
         return map;

docs/changelog/125916.yaml

@@ -0,0 +1,5 @@
+pr: 125916
+summary: Re-enable parallel collection for field sorted top hits
+area: Search
+type: bug
+issues: []

docs/changelog/125930.yaml

@@ -0,0 +1,5 @@
+pr: 125930
+summary: Infer the score mode to use from the Lucene collector
+area: "ES|QL"
+type: enhancement
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -68,6 +68,7 @@
 import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.entitlement.runtime.policy.Platform.LINUX;
+import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.CONFIG;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.DATA;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.SHARED_REPO;
 import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ;
@@ -182,7 +183,8 @@ private static PolicyManager createPolicyManager() {
             FileData.ofPath(bootstrapArgs.libDir(), READ),
             FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE),
             FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
-
+            // exclusive settings file
+            FileData.ofRelativePath(Path.of("operator/settings.json"), CONFIG, READ_WRITE).withExclusive(true),
             // OS release on Linux
             FileData.ofPath(Path.of("/etc/os-release"), READ).withPlatform(LINUX),
             FileData.ofPath(Path.of("/etc/system-release"), READ).withPlatform(LINUX),

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -22,13 +23,16 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.ingest.EnterpriseGeoIpTask;
 import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration;
+import org.elasticsearch.ingest.geoip.direct.DeleteDatabaseConfigurationAction;
 import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.plugins.Plugin;
@@ -38,12 +42,15 @@
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.xcontent.XContentType;
+import org.junit.After;
 import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.ingest.EnterpriseGeoIpTask.ENTERPRISE_GEOIP_DOWNLOADER;
 import static org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderTaskExecutor.IPINFO_TOKEN_SETTING;
@@ -54,6 +61,8 @@ public class EnterpriseGeoIpDownloaderIT extends ESIntegTestCase {
 
     private static final String MAXMIND_DATABASE_TYPE = "GeoIP2-City";
     private static final String IPINFO_DATABASE_TYPE = "asn";
+    private static final String MAXMIND_CONFIGURATION = "test-1";
+    private static final String IPINFO_CONFIGURATION = "test-2";
 
     @ClassRule
     public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture(
@@ -140,6 +149,28 @@ public void testEnterpriseDownloaderTask() throws Exception {
         });
     }
 
+    @After
+    public void cleanup() throws InterruptedException {
+        /*
+         * This method cleans up the database configurations that the test created. This allows the test to be run repeatedly.
+         */
+        CountDownLatch latch = new CountDownLatch(1);
+        LatchedActionListener<AcknowledgedResponse> listener = new LatchedActionListener<>(ActionListener.noop(), latch);
+        SubscribableListener.<AcknowledgedResponse>newForked(l -> deleteDatabaseConfiguration(MAXMIND_CONFIGURATION, l))
+            .<AcknowledgedResponse>andThen(l -> deleteDatabaseConfiguration(IPINFO_CONFIGURATION, l))
+            .addListener(listener);
+        latch.await(10, TimeUnit.SECONDS);
+    }
+
+    private void deleteDatabaseConfiguration(String configurationName, ActionListener<AcknowledgedResponse> listener) {
+        admin().cluster()
+            .execute(
+                DeleteDatabaseConfigurationAction.INSTANCE,
+                new DeleteDatabaseConfigurationAction.Request(TimeValue.MAX_VALUE, TimeValue.timeValueSeconds(10), configurationName),
+                listener
+            );
+    }
+
     private void startEnterpriseGeoIpDownloaderTask() {
         PersistentTasksService persistentTasksService = internalCluster().getInstance(PersistentTasksService.class);
         persistentTasksService.sendStartRequest(
@@ -163,7 +194,7 @@ private void configureMaxmindDatabase(String databaseType) {
                 new PutDatabaseConfigurationAction.Request(
                     TimeValue.MAX_VALUE,
                     TimeValue.MAX_VALUE,
-                    new DatabaseConfiguration("test-1", databaseType, new DatabaseConfiguration.Maxmind("test_account"))
+                    new DatabaseConfiguration(MAXMIND_CONFIGURATION, databaseType, new DatabaseConfiguration.Maxmind("test_account"))
                 )
             )
             .actionGet();
@@ -176,7 +207,7 @@ private void configureIpinfoDatabase(String databaseType) {
                 new PutDatabaseConfigurationAction.Request(
                     TimeValue.MAX_VALUE,
                     TimeValue.MAX_VALUE,
-                    new DatabaseConfiguration("test-2", databaseType, new DatabaseConfiguration.Ipinfo())
+                    new DatabaseConfiguration(IPINFO_CONFIGURATION, databaseType, new DatabaseConfiguration.Ipinfo())
                 )
             )
             .actionGet();

muted-tests.yml

@@ -344,18 +344,12 @@ tests:
 - class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
   method: testSearchableSnapshotsInHotPhasePinnedToHotNodes
   issue: https://github.com/elastic/elasticsearch/issues/125683
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=cat.allocation/10_basic/Node forecasts}
-  issue: https://github.com/elastic/elasticsearch/issues/125711
 - class: org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportActionIT
   method: testAlreadyUpToDateDataStream
   issue: https://github.com/elastic/elasticsearch/issues/125727
 - class: org.elasticsearch.index.engine.ThreadPoolMergeSchedulerStressTestIT
   method: testMergingFallsBehindAndThenCatchesUp
   issue: https://github.com/elastic/elasticsearch/issues/125744
-- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
-  method: test {yaml=cat.allocation/10_basic/Node forecasts}
-  issue: https://github.com/elastic/elasticsearch/issues/125661
 - class: org.elasticsearch.xpack.esql.spatial.SpatialExtentAggregationNoLicenseIT
   method: testStExtentAggregationWithPoints
   issue: https://github.com/elastic/elasticsearch/issues/125735
@@ -380,9 +374,6 @@ tests:
 - class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=search.vectors/41_knn_search_bbq_hnsw/Test index configured rescore vector score consistency}
   issue: https://github.com/elastic/elasticsearch/issues/125902
-- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
-  method: test {yaml=cat.allocation/10_basic/Node forecasts}
-  issue: https://github.com/elastic/elasticsearch/issues/125848
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/start_data_frame_analytics/Test start given dest index is not empty}
   issue: https://github.com/elastic/elasticsearch/issues/125909
@@ -398,6 +389,12 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_stats/Test get transform stats with timeout}
   issue: https://github.com/elastic/elasticsearch/issues/125975
+- class: org.elasticsearch.search.SearchServiceSingleNodeTests
+  method: testSlicingBehaviourForParallelCollection
+  issue: https://github.com/elastic/elasticsearch/issues/125899
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test021InstallPlugin
+  issue: https://github.com/elastic/elasticsearch/issues/116147
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.allocation/10_basic.yml

@@ -295,7 +295,7 @@
           ( [-\w.]+                \s+
             [-\w.]+                \s+
             [-\w.]+                \s+
-            [\w]+
+            [-\w.]+
             \n
           )+
         $/

server/src/main/java/org/elasticsearch/action/ResolvedIndices.java

@@ -11,13 +11,15 @@
 
 import org.elasticsearch.action.search.SearchContextId;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.RemoteClusterService;
@@ -176,6 +178,20 @@ public static ResolvedIndices resolveWithIndexNamesAndOptions(
             ? Index.EMPTY_ARRAY
             : indexNameExpressionResolver.concreteIndices(projectMetadata, localIndices, startTimeInMillis);
 
+        // prevent using selectors with remote cluster patterns
+        if (DataStream.isFailureStoreFeatureFlagEnabled()) {
+            for (final var indicesPerRemoteClusterAlias : remoteClusterIndices.entrySet()) {
+                final String[] indices = indicesPerRemoteClusterAlias.getValue().indices();
+                if (indices != null) {
+                    for (final String index : indices) {
+                        if (IndexNameExpressionResolver.hasSelectorSuffix(index)) {
+                            throw new InvalidIndexNameException(index, "Selectors are not yet supported on remote cluster patterns");
+                        }
+                    }
+                }
+            }
+        }
+
         return new ResolvedIndices(
             remoteClusterIndices,
             localIndices,

server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java

@@ -16,15 +16,16 @@
 import org.elasticsearch.core.FixForMultiProject;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.ClosedWatchServiceException;
-import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.StandardWatchEventKinds;
 import java.nio.file.WatchEvent;
 import java.nio.file.WatchKey;
 import java.nio.file.WatchService;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -62,8 +63,9 @@ public abstract class AbstractFileWatchingService extends AbstractLifecycleCompo
     private WatchKey settingsDirWatchKey;
     private WatchKey configDirWatchKey;
 
+    @SuppressWarnings("this-escape")
     public AbstractFileWatchingService(Path settingsDir) {
-        if (Files.exists(settingsDir) && Files.isDirectory(settingsDir) == false) {
+        if (filesExists(settingsDir) && filesIsDirectory(settingsDir) == false) {
             throw new IllegalArgumentException("settingsDir should be a directory");
         }
         this.settingsDir = settingsDir;
@@ -113,10 +115,10 @@ public final boolean watching() {
         return watcherThread != null;
     }
 
-    private static FileUpdateState readFileUpdateState(Path path) throws IOException {
+    private FileUpdateState readFileUpdateState(Path path) throws IOException {
         BasicFileAttributes attr;
         try {
-            attr = Files.readAttributes(path, BasicFileAttributes.class);
+            attr = filesReadAttributes(path, BasicFileAttributes.class);
         } catch (NoSuchFileException e) {
             // file doesn't exist anymore
             return null;
@@ -141,7 +143,7 @@ final boolean fileChanged(Path path) throws IOException {
     }
 
     protected final synchronized void startWatcher() {
-        if (Files.exists(settingsDir.getParent()) == false) {
+        if (filesExists(settingsDir.getParent()) == false) {
             logger.warn("File watcher for [{}] cannot start because parent directory does not exist", settingsDir);
             return;
         }
@@ -155,7 +157,7 @@ protected final synchronized void startWatcher() {
          */
         try {
             this.watchService = settingsDir.getParent().getFileSystem().newWatchService();
-            if (Files.exists(settingsDir)) {
+            if (filesExists(settingsDir)) {
                 settingsDirWatchKey = enableDirectoryWatcher(settingsDirWatchKey, settingsDir);
             } else {
                 logger.debug("watched directory [{}] not found, will watch for its creation...", settingsDir);
@@ -188,8 +190,8 @@ protected final void watcherThread() {
         try {
             logger.info("file settings service up and running [tid={}]", Thread.currentThread().getId());
 
-            if (Files.exists(settingsDir)) {
-                try (Stream<Path> files = Files.list(settingsDir)) {
+            if (filesExists(settingsDir)) {
+                try (Stream<Path> files = filesList(settingsDir)) {
                     var f = files.iterator();
                     if (f.hasNext() == false) {
                         // no files in directory
@@ -248,7 +250,7 @@ protected final void watcherThread() {
                         }
                     }
                 } else if (key == configDirWatchKey) {
-                    if (Files.exists(settingsDir)) {
+                    if (filesExists(settingsDir)) {
                         // We re-register the settings directory watch key, because we don't know
                         // if the file name maps to the same native file system file id. Symlinks
                         // are one potential cause of inconsistency here, since their handling by
@@ -257,7 +259,7 @@ protected final void watcherThread() {
                         settingsDirWatchKey = enableDirectoryWatcher(settingsDirWatchKey, settingsDir);
 
                         // re-read the settings directory, and ping for any changes
-                        try (Stream<Path> files = Files.list(settingsDir)) {
+                        try (Stream<Path> files = filesList(settingsDir)) {
                             for (var f = files.iterator(); f.hasNext();) {
                                 Path file = f.next();
                                 if (fileChanged(file)) {
@@ -370,4 +372,19 @@ long retryDelayMillis(int failedCount) {
      * class to determine if a file has been changed.
      */
     private record FileUpdateState(long timestamp, String path, Object fileKey) {}
+
+    // the following methods are a workaround to ensure exclusive access for files
+    // required by child watchers; this is required because we only check the caller's module
+    // not the entire stack
+    protected abstract boolean filesExists(Path path);
+
+    protected abstract boolean filesIsDirectory(Path path);
+
+    protected abstract <A extends BasicFileAttributes> A filesReadAttributes(Path path, Class<A> clazz) throws IOException;
+
+    protected abstract Stream<Path> filesList(Path dir) throws IOException;
+
+    protected abstract Path filesSetLastModifiedTime(Path path, FileTime time) throws IOException;
+
+    protected abstract InputStream filesNewInputStream(Path path) throws IOException;
 }