Merge remote-tracking branch 'upstream/main' into ES-10063-indices-stats-mp

Author: PeteGillinElastic

docs/changelog/127877.yaml

@@ -0,0 +1,5 @@
+pr: 127877
+summary: Check hidden frames in entitlements
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/127921.yaml

@@ -0,0 +1,5 @@
+pr: 127921
+summary: "[9.x] Revert \"Enable madvise by default for all builds\""
+area: Vector Search
+type: bug
+issues: []

libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/Util.java

@@ -10,8 +10,10 @@
 package org.elasticsearch.entitlement.bridge;
 
 import java.util.Optional;
+import java.util.Set;
 
 import static java.lang.StackWalker.Option.RETAIN_CLASS_REFERENCE;
+import static java.lang.StackWalker.Option.SHOW_HIDDEN_FRAMES;
 
 public class Util {
     /**
@@ -23,6 +25,8 @@ public class Util {
     public static final Class<?> NO_CLASS = new Object() {
     }.getClass();
 
+    private static final Set<String> skipInternalPackages = Set.of("java.lang.invoke", "java.lang.reflect", "jdk.internal.reflect");
+
     /**
      * Why would we write this instead of using {@link StackWalker#getCallerClass()}?
      * Because that method throws {@link IllegalCallerException} if called from the "outermost frame",
@@ -32,9 +36,10 @@ public class Util {
      */
     @SuppressWarnings("unused") // Called reflectively from InstrumenterImpl
     public static Class<?> getCallerClass() {
-        Optional<Class<?>> callerClassIfAny = StackWalker.getInstance(RETAIN_CLASS_REFERENCE)
+        Optional<Class<?>> callerClassIfAny = StackWalker.getInstance(Set.of(RETAIN_CLASS_REFERENCE, SHOW_HIDDEN_FRAMES))
             .walk(
                 frames -> frames.skip(2) // Skip this method and its caller
+                    .filter(frame -> skipInternalPackages.contains(frame.getDeclaringClass().getPackageName()) == false)
                     .findFirst()
                     .map(StackWalker.StackFrame::getDeclaringClass)
             );

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java

@@ -182,7 +182,9 @@ private static BaseDir parseBaseDir(String baseDir) {
             case "config" -> BaseDir.CONFIG;
             case "data" -> BaseDir.DATA;
             case "home" -> BaseDir.USER_HOME;
-            // NOTE: shared_repo is _not_ accessible to policy files, only internally
+            // it would be nice to limit this to just ES modules, but we don't have a way to plumb that through to here
+            // however, we still don't document in the error case below that shared_repo is valid
+            case "shared_repo" -> BaseDir.SHARED_REPO;
             default -> throw new PolicyValidationException(
                 "invalid relative directory: " + baseDir + ", valid values: [config, data, home]"
             );

modules/repository-url/src/main/plugin-metadata/entitlement-policy.yaml

@@ -1,2 +1,8 @@
+org.elasticsearch.repository.url:
+  - outbound_network
+  - files:
+      - relative_path: .
+        relative_to: shared_repo
+        mode: read
 org.apache.httpcomponents.httpclient:
   - outbound_network # for URLHttpClient

muted-tests.yml

@@ -66,9 +66,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test start already started transform}
   issue: https://github.com/elastic/elasticsearch/issues/98802
-- class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT
-  method: testDeprecatedSettingsReturnWarnings
-  issue: https://github.com/elastic/elasticsearch/issues/108628
 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
   method: testAllocationPreventedForRemoval
   issue: https://github.com/elastic/elasticsearch/issues/116363
@@ -408,9 +405,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
   method: testLookupExplosionNoFetch
   issue: https://github.com/elastic/elasticsearch/issues/127365
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
-  method: test
-  issue: https://github.com/elastic/elasticsearch/issues/127536
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header}
   issue: https://github.com/elastic/elasticsearch/issues/127625
@@ -462,6 +456,15 @@ tests:
 - class: org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT
   method: testRestoreSnapshotAllocationDoesNotExceedWatermarkWithMultipleRestores
   issue: https://github.com/elastic/elasticsearch/issues/127787
+- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithFiltersIT
+  method: testTimestampFilterFromQuery
+  issue: https://github.com/elastic/elasticsearch/issues/127332
+- class: org.elasticsearch.indices.stats.IndexStatsIT
+  method: testThrottleStats
+  issue: https://github.com/elastic/elasticsearch/issues/126359
+- class: org.elasticsearch.search.vectors.IVFKnnFloatVectorQueryTests
+  method: testRandomWithFilter
+  issue: https://github.com/elastic/elasticsearch/issues/127963
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -252,6 +252,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion FIELD_CAPS_ADD_CLUSTER_ALIAS = def(9_073_0_00);
     public static final TransportVersion INFERENCE_ADD_TIMEOUT_PUT_ENDPOINT = def(9_074_0_00);
     public static final TransportVersion ESQL_FIELD_ATTRIBUTE_DROP_TYPE = def(9_075_0_00);
+    public static final TransportVersion ESQL_TIME_SERIES_SOURCE_STATUS = def(9_076_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/service/MasterService.java

@@ -913,7 +913,7 @@ public void onFailure(Exception failure) {
 
         @Override
         public Releasable captureResponseHeaders() {
-            final var storedContext = threadContext.newStoredContext();
+            final var storedContext = threadContextSupplier.get();
             return Releasables.wrap(() -> {
                 final var newResponseHeaders = threadContext.getResponseHeaders();
                 if (newResponseHeaders.isEmpty()) {

server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java

@@ -20,9 +20,11 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.NativeFSLockFactory;
+import org.apache.lucene.store.ReadAdvice;
 import org.apache.lucene.store.SimpleFSLockFactory;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
@@ -43,6 +45,7 @@
 public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory {
 
     private static final Logger Log = LogManager.getLogger(FsDirectoryFactory.class);
+    private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random");
 
     public static final Setting<LockFactory> INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> {
         return switch (s) {
@@ -75,12 +78,20 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
                 // Use Lucene defaults
                 final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
                 if (primaryDirectory instanceof MMapDirectory mMapDirectory) {
-                    return new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions));
+                    Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions));
+                    if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
+                        dir = disableRandomAdvice(dir);
+                    }
+                    return dir;
                 } else {
                     return primaryDirectory;
                 }
             case MMAPFS:
-                return setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions);
+                Directory dir = setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions);
+                if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
+                    dir = disableRandomAdvice(dir);
+                }
+                return dir;
             case SIMPLEFS:
             case NIOFS:
                 return new NIOFSDirectory(location, lockFactory);
@@ -108,6 +119,23 @@ static BiPredicate<String, IOContext> getPreloadFunc(Set<String> preLoadExtensio
         return MMapDirectory.NO_FILES;
     }
 
+    /**
+     * Return a {@link FilterDirectory} around the provided {@link Directory} that forcefully disables {@link IOContext#readAdvice random
+     * access}.
+     */
+    static Directory disableRandomAdvice(Directory dir) {
+        return new FilterDirectory(dir) {
+            @Override
+            public IndexInput openInput(String name, IOContext context) throws IOException {
+                if (context.readAdvice() == ReadAdvice.RANDOM) {
+                    context = context.withReadAdvice(ReadAdvice.NORMAL);
+                }
+                assert context.readAdvice() != ReadAdvice.RANDOM;
+                return super.openInput(name, context);
+            }
+        };
+    }
+
     /**
      * Returns true iff the directory is a hybrid fs directory
      */

server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java

@@ -299,7 +299,6 @@ public void testThreadContext() {
             }
 
             try (ThreadContext.StoredContext ignored = threadPool.getThreadContext().stashContext()) {
-
                 final var expectedHeaders = new HashMap<String, String>();
                 expectedHeaders.put(randomIdentifier(), randomIdentifier());
                 for (final var copiedHeader : Task.HEADERS_TO_COPY) {
@@ -319,8 +318,10 @@ public void testThreadContext() {
                     new AckedClusterStateUpdateTask(ackedRequest(ackTimeout, masterTimeout), null) {
                         @Override
                         public ClusterState execute(ClusterState currentState) {
-                            assertTrue(threadPool.getThreadContext().isSystemContext());
-                            assertEquals(Collections.emptyMap(), threadPool.getThreadContext().getHeaders());
+                            // the task is executed in the context in which the task was originally created.
+                            // note: this is typically not a system context.
+                            assertExpectedThreadContext(Map.of());
+
                             expectedResponseHeaders.forEach(
                                 (name, values) -> values.forEach(v -> threadPool.getThreadContext().addResponseHeader(name, v))
                             );