Merge branch 'main' into fix-sampletests

Author: jan-elastic

docs/changelog/127921.yaml

@@ -0,0 +1,5 @@
+pr: 127921
+summary: "[9.x] Revert \"Enable madvise by default for all builds\""
+area: Vector Search
+type: bug
+issues: []

muted-tests.yml

@@ -462,6 +462,9 @@ tests:
 - class: org.elasticsearch.indices.stats.IndexStatsIT
   method: testThrottleStats
   issue: https://github.com/elastic/elasticsearch/issues/126359
+- class: org.elasticsearch.search.vectors.IVFKnnFloatVectorQueryTests
+  method: testRandomWithFilter
+  issue: https://github.com/elastic/elasticsearch/issues/127963
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java

@@ -20,9 +20,11 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.NativeFSLockFactory;
+import org.apache.lucene.store.ReadAdvice;
 import org.apache.lucene.store.SimpleFSLockFactory;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
@@ -43,6 +45,7 @@
 public class FsDirectoryFactory implements IndexStorePlugin.DirectoryFactory {
 
     private static final Logger Log = LogManager.getLogger(FsDirectoryFactory.class);
+    private static final FeatureFlag MADV_RANDOM_FEATURE_FLAG = new FeatureFlag("madv_random");
 
     public static final Setting<LockFactory> INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> {
         return switch (s) {
@@ -75,12 +78,20 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
                 // Use Lucene defaults
                 final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
                 if (primaryDirectory instanceof MMapDirectory mMapDirectory) {
-                    return new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions));
+                    Directory dir = new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions));
+                    if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
+                        dir = disableRandomAdvice(dir);
+                    }
+                    return dir;
                 } else {
                     return primaryDirectory;
                 }
             case MMAPFS:
-                return setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions);
+                Directory dir = setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions);
+                if (MADV_RANDOM_FEATURE_FLAG.isEnabled() == false) {
+                    dir = disableRandomAdvice(dir);
+                }
+                return dir;
             case SIMPLEFS:
             case NIOFS:
                 return new NIOFSDirectory(location, lockFactory);
@@ -108,6 +119,23 @@ static BiPredicate<String, IOContext> getPreloadFunc(Set<String> preLoadExtensio
         return MMapDirectory.NO_FILES;
     }
 
+    /**
+     * Return a {@link FilterDirectory} around the provided {@link Directory} that forcefully disables {@link IOContext#readAdvice random
+     * access}.
+     */
+    static Directory disableRandomAdvice(Directory dir) {
+        return new FilterDirectory(dir) {
+            @Override
+            public IndexInput openInput(String name, IOContext context) throws IOException {
+                if (context.readAdvice() == ReadAdvice.RANDOM) {
+                    context = context.withReadAdvice(ReadAdvice.NORMAL);
+                }
+                assert context.readAdvice() != ReadAdvice.RANDOM;
+                return super.openInput(name, context);
+            }
+        };
+    }
+
     /**
      * Returns true iff the directory is a hybrid fs directory
      */

server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java

@@ -9,12 +9,16 @@
 package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.NoLockFactory;
+import org.apache.lucene.store.ReadAdvice;
 import org.apache.lucene.store.SleepingLockWrapper;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -34,6 +38,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
@@ -74,6 +79,29 @@ public void testPreload() throws IOException {
         }
     }
 
+    public void testDisableRandomAdvice() throws IOException {
+        Directory dir = new FilterDirectory(new ByteBuffersDirectory()) {
+            @Override
+            public IndexInput openInput(String name, IOContext context) throws IOException {
+                assertFalse(context.readAdvice() == ReadAdvice.RANDOM);
+                return super.openInput(name, context);
+            }
+        };
+        Directory noRandomAccessDir = FsDirectoryFactory.disableRandomAdvice(dir);
+        try (IndexOutput out = noRandomAccessDir.createOutput("foo", IOContext.DEFAULT)) {
+            out.writeInt(42);
+        }
+        // Test the tester
+        expectThrows(AssertionError.class, () -> dir.openInput("foo", IOContext.DEFAULT.withReadAdvice(ReadAdvice.RANDOM)));
+
+        // The wrapped directory shouldn't fail regardless of the IOContext
+        for (IOContext context : List.of(IOContext.DEFAULT, IOContext.READONCE, IOContext.DEFAULT.withReadAdvice(ReadAdvice.RANDOM))) {
+            try (IndexInput in = noRandomAccessDir.openInput("foo", context)) {
+                assertEquals(42, in.readInt());
+            }
+        }
+    }
+
     private Directory newDirectory(Settings settings) throws IOException {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("foo", settings);
         Path tempDir = createTempDir().resolve(idxSettings.getUUID()).resolve("0");

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplication.java

@@ -12,11 +12,10 @@
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
@@ -282,15 +281,13 @@ public String toString() {
      * Returns the merged {@link SearchApplication} from the current state and the provided {@param update}.
      * This function returns the current instance if the update is a noop.
      *
-     * @param update The source of the update represented in bytes.
+     * @param update       The source of the update represented in bytes.
      * @param xContentType The format of the bytes.
-     * @param bigArrays The {@link BigArrays} to use to recycle bytes array.
-     *
      * @return The merged {@link SearchApplication}.
      */
-    SearchApplication merge(BytesReference update, XContentType xContentType, BigArrays bigArrays) throws IOException {
+    SearchApplication merge(BytesReference update, XContentType xContentType) throws IOException {
         final Tuple<XContentType, Map<String, Object>> sourceAndContent;
-        try (ReleasableBytesStreamOutput sourceBuffer = new ReleasableBytesStreamOutput(0, bigArrays.withCircuitBreaking())) {
+        try (BytesStreamOutput sourceBuffer = new BytesStreamOutput()) {
             try (XContentBuilder builder = XContentFactory.jsonBuilder(sourceBuffer)) {
                 toXContent(builder, EMPTY_PARAMS);
             }
@@ -305,7 +302,7 @@ SearchApplication merge(BytesReference update, XContentType xContentType, BigArr
             return this;
         }
 
-        try (ReleasableBytesStreamOutput newSourceBuffer = new ReleasableBytesStreamOutput(0, bigArrays.withCircuitBreaking())) {
+        try (BytesStreamOutput newSourceBuffer = new BytesStreamOutput()) {
             try (XContentBuilder builder = XContentFactory.jsonBuilder(newSourceBuffer)) {
                 builder.value(newSourceAsMap);
             }

x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/SearchApplicationTests.java

@@ -16,7 +16,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
@@ -111,7 +110,7 @@ public void testMerge() throws IOException {
               "updated_at_millis": 12345
             }""";
         SearchApplication app = SearchApplication.fromXContentBytes("my_search_app", new BytesArray(content), XContentType.JSON);
-        SearchApplication updatedApp = app.merge(new BytesArray(update), XContentType.JSON, BigArrays.NON_RECYCLING_INSTANCE);
+        SearchApplication updatedApp = app.merge(new BytesArray(update), XContentType.JSON);
         assertNotSame(app, updatedApp);
         assertThat(updatedApp.indices(), equalTo(new String[] { "my_index", "my_index_2" }));
         assertThat(updatedApp.analyticsCollectionName(), equalTo("my_search_app_analytics"));