diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 5843c562dd5ff..b496e6d861ca6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -163,7 +163,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_TO_LUCENE_10_2_1 = def(9_023_00_0, Version.LUCENE_10_2_1); public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = def(9_024_0_00, Version.LUCENE_10_2_1); public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_025_0_00, Version.LUCENE_10_2_1); - + public static final IndexVersion UPGRADE_TO_LUCENE_10_3_0 = def(9_050_00_0, Version.LUCENE_10_3_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOIndexInputSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOHint.java similarity index 59% rename from server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOIndexInputSupplier.java rename to server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOHint.java index 0640a5dacce65..73b7182911114 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOIndexInputSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOHint.java @@ -10,14 +10,7 @@ package org.elasticsearch.index.codec.vectors.es818; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import java.io.IOException; - -/** - * A hook for {@link DirectIOLucene99FlatVectorsReader} to specify the input should be opened using DirectIO. - * Remove when IOContext allows more extensible payloads to be specified. - */ -public interface DirectIOIndexInputSupplier { - IndexInput openInputDirect(String name, IOContext context) throws IOException; +public enum DirectIOHint implements IOContext.FileOpenHint { + INSTANCE } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java index 02de4d5450d35..a0cd6dbf65688 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsFormat.java @@ -27,8 +27,14 @@ import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.FlushInfo; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.MergeInfo; +import org.apache.lucene.store.ReadAdvice; import java.io.IOException; +import java.util.Optional; +import java.util.Set; /** * Copied from Lucene99FlatVectorsFormat in Lucene 10.1 @@ -61,22 +67,62 @@ public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOExceptio return new Lucene99FlatVectorsWriter(state, vectorsScorer); } + private static final IOContext DIRECT_IO_CONTEXT = new IOContext() { + @Override + public Context context() { + return Context.DEFAULT; + } + + @Override + public MergeInfo mergeInfo() { + return null; + } + + @Override + public FlushInfo flushInfo() { + return null; + } + + @Override + public Set hints() { + return Set.of(DirectIOHint.INSTANCE); + } + + @Override + public IOContext withHints(FileOpenHint... hints) { + return this; + } + + @Override + public Optional readAdvice() { + return Optional.empty(); + } + + @Override + public IOContext withReadAdvice(ReadAdvice advice) { + return this; + } + }; + @Override public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { - if (DirectIOLucene99FlatVectorsReader.shouldUseDirectIO(state)) { - // Use mmap for merges and direct I/O for searches. - // TODO: Open the mmap file with sequential access instead of random (current behavior). - return new MergeReaderWrapper( - new DirectIOLucene99FlatVectorsReader(state, vectorsScorer), - new Lucene99FlatVectorsReader(state, vectorsScorer) - ); - } else { - return new Lucene99FlatVectorsReader(state, vectorsScorer); - } + SegmentReadState directIOState = new SegmentReadState( + state.directory, + state.segmentInfo, + state.fieldInfos, + DIRECT_IO_CONTEXT, + state.segmentSuffix + ); + // Use mmap for merges and direct I/O for searches. + // TODO: Open the mmap file with sequential access instead of random (current behavior). + return new MergeReaderWrapper( + new Lucene99FlatVectorsReader(directIOState, vectorsScorer), + new Lucene99FlatVectorsReader(state, vectorsScorer) + ); } @Override public String toString() { - return "ES818FlatVectorsFormat(" + "vectorsScorer=" + vectorsScorer + ')'; + return "Lucene99FlatVectorsFormat(" + "vectorsScorer=" + vectorsScorer + ')'; } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java deleted file mode 100644 index 0d087cb5d73c9..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/DirectIOLucene99FlatVectorsReader.java +++ /dev/null @@ -1,358 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2025 Elasticsearch B.V. - */ -package org.elasticsearch.index.codec.vectors.es818; - -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.hnsw.FlatVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; -import org.apache.lucene.codecs.lucene95.OffHeapByteVectorValues; -import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues; -import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; -import org.apache.lucene.index.ByteVectorValues; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.FloatVectorValues; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.VectorEncoding; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.internal.hppc.IntObjectHashMap; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.ReadAdvice; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.RamUsageEstimator; -import org.apache.lucene.util.SuppressForbidden; -import org.apache.lucene.util.hnsw.RandomVectorScorer; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Map; - -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; - -/** Copied from Lucene99FlatVectorsReader in Lucene 10.2, then modified to support DirectIOIndexInputSupplier */ -@SuppressForbidden(reason = "Copied from lucene") -public class DirectIOLucene99FlatVectorsReader extends FlatVectorsReader { - - private static final boolean USE_DIRECT_IO = Boolean.parseBoolean(System.getProperty("vector.rescoring.directio", "true")); - - private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(DirectIOLucene99FlatVectorsReader.class); - - private final IntObjectHashMap fields = new IntObjectHashMap<>(); - private final IndexInput vectorData; - private final FieldInfos fieldInfos; - - public DirectIOLucene99FlatVectorsReader(SegmentReadState state, FlatVectorsScorer scorer) throws IOException { - super(scorer); - int versionMeta = readMetadata(state); - this.fieldInfos = state.fieldInfos; - boolean success = false; - try { - vectorData = openDataInput( - state, - versionMeta, - DirectIOLucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION, - DirectIOLucene99FlatVectorsFormat.VECTOR_DATA_CODEC_NAME, - // Flat formats are used to randomly access vectors from their node ID that is stored - // in the HNSW graph. - state.context.withReadAdvice(ReadAdvice.RANDOM) - ); - success = true; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(this); - } - } - } - - public static boolean shouldUseDirectIO(SegmentReadState state) { - return USE_DIRECT_IO && state.directory instanceof DirectIOIndexInputSupplier; - } - - private int readMetadata(SegmentReadState state) throws IOException { - String metaFileName = IndexFileNames.segmentFileName( - state.segmentInfo.name, - state.segmentSuffix, - DirectIOLucene99FlatVectorsFormat.META_EXTENSION - ); - int versionMeta = -1; - try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) { - Throwable priorE = null; - try { - versionMeta = CodecUtil.checkIndexHeader( - meta, - DirectIOLucene99FlatVectorsFormat.META_CODEC_NAME, - DirectIOLucene99FlatVectorsFormat.VERSION_START, - DirectIOLucene99FlatVectorsFormat.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - readFields(meta, state.fieldInfos); - } catch (Throwable exception) { - priorE = exception; - } finally { - CodecUtil.checkFooter(meta, priorE); - } - } - return versionMeta; - } - - private static IndexInput openDataInput( - SegmentReadState state, - int versionMeta, - String fileExtension, - String codecName, - IOContext context - ) throws IOException { - String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension); - // use direct IO for accessing raw vector data for searches - IndexInput in = USE_DIRECT_IO && state.directory instanceof DirectIOIndexInputSupplier did - ? did.openInputDirect(fileName, context) - : state.directory.openInput(fileName, context); - boolean success = false; - try { - int versionVectorData = CodecUtil.checkIndexHeader( - in, - codecName, - DirectIOLucene99FlatVectorsFormat.VERSION_START, - DirectIOLucene99FlatVectorsFormat.VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - if (versionMeta != versionVectorData) { - throw new CorruptIndexException( - "Format versions mismatch: meta=" + versionMeta + ", " + codecName + "=" + versionVectorData, - in - ); - } - CodecUtil.retrieveChecksum(in); - success = true; - return in; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(in); - } - } - } - - private void readFields(ChecksumIndexInput meta, FieldInfos infos) throws IOException { - for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { - FieldInfo info = infos.fieldInfo(fieldNumber); - if (info == null) { - throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); - } - FieldEntry fieldEntry = FieldEntry.create(meta, info); - fields.put(info.number, fieldEntry); - } - } - - @Override - public long ramBytesUsed() { - return SHALLOW_SIZE + fields.ramBytesUsed(); - } - - @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - final FieldEntry entry = getFieldEntryOrThrow(fieldInfo.name); - return Map.of(DirectIOLucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION, entry.vectorDataLength()); - } - - @Override - public void checkIntegrity() throws IOException { - CodecUtil.checksumEntireFile(vectorData); - } - - @Override - public FlatVectorsReader getMergeInstance() { - try { - // Update the read advice since vectors are guaranteed to be accessed sequentially for merge - this.vectorData.updateReadAdvice(ReadAdvice.SEQUENTIAL); - return this; - } catch (IOException exception) { - throw new UncheckedIOException(exception); - } - } - - private FieldEntry getFieldEntryOrThrow(String field) { - final FieldInfo info = fieldInfos.fieldInfo(field); - final FieldEntry entry; - if (info == null || (entry = fields.get(info.number)) == null) { - throw new IllegalArgumentException("field=\"" + field + "\" not found"); - } - return entry; - } - - private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) { - final FieldEntry fieldEntry = getFieldEntryOrThrow(field); - if (fieldEntry.vectorEncoding != expectedEncoding) { - throw new IllegalArgumentException( - "field=\"" + field + "\" is encoded as: " + fieldEntry.vectorEncoding + " expected: " + expectedEncoding - ); - } - return fieldEntry; - } - - @Override - public FloatVectorValues getFloatVectorValues(String field) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field, VectorEncoding.FLOAT32); - return OffHeapFloatVectorValues.load( - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.ordToDoc, - fieldEntry.vectorEncoding, - fieldEntry.dimension, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - vectorData - ); - } - - @Override - public ByteVectorValues getByteVectorValues(String field) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field, VectorEncoding.BYTE); - return OffHeapByteVectorValues.load( - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.ordToDoc, - fieldEntry.vectorEncoding, - fieldEntry.dimension, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - vectorData - ); - } - - @Override - public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field, VectorEncoding.FLOAT32); - return vectorScorer.getRandomVectorScorer( - fieldEntry.similarityFunction, - OffHeapFloatVectorValues.load( - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.ordToDoc, - fieldEntry.vectorEncoding, - fieldEntry.dimension, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - vectorData - ), - target - ); - } - - @Override - public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field, VectorEncoding.BYTE); - return vectorScorer.getRandomVectorScorer( - fieldEntry.similarityFunction, - OffHeapByteVectorValues.load( - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.ordToDoc, - fieldEntry.vectorEncoding, - fieldEntry.dimension, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - vectorData - ), - target - ); - } - - @Override - public void finishMerge() throws IOException { - // This makes sure that the access pattern hint is reverted back since HNSW implementation - // needs it - this.vectorData.updateReadAdvice(ReadAdvice.RANDOM); - } - - @Override - public void close() throws IOException { - IOUtils.close(vectorData); - } - - private record FieldEntry( - VectorSimilarityFunction similarityFunction, - VectorEncoding vectorEncoding, - long vectorDataOffset, - long vectorDataLength, - int dimension, - int size, - OrdToDocDISIReaderConfiguration ordToDoc, - FieldInfo info - ) { - - FieldEntry { - if (similarityFunction != info.getVectorSimilarityFunction()) { - throw new IllegalStateException( - "Inconsistent vector similarity function for field=\"" - + info.name - + "\"; " - + similarityFunction - + " != " - + info.getVectorSimilarityFunction() - ); - } - int infoVectorDimension = info.getVectorDimension(); - if (infoVectorDimension != dimension) { - throw new IllegalStateException( - "Inconsistent vector dimension for field=\"" + info.name + "\"; " + infoVectorDimension + " != " + dimension - ); - } - - int byteSize = switch (info.getVectorEncoding()) { - case BYTE -> Byte.BYTES; - case FLOAT32 -> Float.BYTES; - }; - long vectorBytes = Math.multiplyExact((long) infoVectorDimension, byteSize); - long numBytes = Math.multiplyExact(vectorBytes, size); - if (numBytes != vectorDataLength) { - throw new IllegalStateException( - "Vector data length " - + vectorDataLength - + " not matching size=" - + size - + " * dim=" - + dimension - + " * byteSize=" - + byteSize - + " = " - + numBytes - ); - } - } - - static FieldEntry create(IndexInput input, FieldInfo info) throws IOException { - final VectorEncoding vectorEncoding = readVectorEncoding(input); - final VectorSimilarityFunction similarityFunction = readSimilarityFunction(input); - final var vectorDataOffset = input.readVLong(); - final var vectorDataLength = input.readVLong(); - final var dimension = input.readVInt(); - final var size = input.readInt(); - final var ordToDoc = OrdToDocDISIReaderConfiguration.fromStoredMeta(input, size); - return new FieldEntry(similarityFunction, vectorEncoding, vectorDataOffset, vectorDataLength, dimension, size, ordToDoc, info); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java index c3e12a765eb5f..3ab49ab590346 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormat.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer; @@ -86,6 +87,8 @@ */ public class ES818BinaryQuantizedVectorsFormat extends FlatVectorsFormat { + private static final boolean USE_DIRECT_IO = Boolean.parseBoolean(System.getProperty("vector.rescoring.directio", "true")); + public static final String BINARIZED_VECTOR_COMPONENT = "BVEC"; public static final String NAME = "ES818BinaryQuantizedVectorsFormat"; @@ -97,9 +100,9 @@ public class ES818BinaryQuantizedVectorsFormat extends FlatVectorsFormat { static final String VECTOR_DATA_EXTENSION = "veb"; static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16; - private static final DirectIOLucene99FlatVectorsFormat rawVectorFormat = new DirectIOLucene99FlatVectorsFormat( - FlatVectorScorerUtil.getLucene99FlatVectorsScorer() - ); + private static final FlatVectorsFormat rawVectorFormat = USE_DIRECT_IO + ? new DirectIOLucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer()) + : new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer()); private static final ES818BinaryFlatVectorsScorer scorer = new ES818BinaryFlatVectorsScorer( FlatVectorScorerUtil.getLucene99FlatVectorsScorer() diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java index eaafbe7bfc947..eef9f5cc28b07 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java @@ -11,6 +11,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.util.Accountable; @@ -20,6 +21,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Map; class MergeReaderWrapper extends FlatVectorsReader { @@ -57,6 +59,16 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { return mainReader.getByteVectorValues(field); } + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + mainReader.search(field, target, knnCollector, acceptDocs); + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + mainReader.search(field, target, knnCollector, acceptDocs); + } + @Override public FlatVectorsReader getMergeInstance() { return mergeReader; @@ -73,13 +85,8 @@ public Collection getChildResources() { } @Override - public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { - mainReader.search(field, target, knnCollector, acceptDocs); - } - - @Override - public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { - mainReader.search(field, target, knnCollector, acceptDocs); + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return mainReader.getOffHeapByteSize(fieldInfo); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java index 76dfee2f7f2de..bc1d469ab3486 100644 --- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java @@ -26,7 +26,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.codec.vectors.es818.DirectIOIndexInputSupplier; +import org.elasticsearch.index.codec.vectors.es818.DirectIOHint; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -116,7 +116,7 @@ public static boolean isHybridFs(Directory directory) { return unwrap instanceof HybridDirectory; } - static final class HybridDirectory extends NIOFSDirectory implements DirectIOIndexInputSupplier { + static final class HybridDirectory extends NIOFSDirectory { private final MMapDirectory delegate; private final DirectIODirectory directIODelegate; @@ -143,7 +143,11 @@ protected boolean useDirectIO(String name, IOContext context, OptionalLong fileL @Override public IndexInput openInput(String name, IOContext context) throws IOException { - if (useDelegate(name, context)) { + if (directIODelegate != null && context.hints().contains(DirectIOHint.INSTANCE)) { + ensureOpen(); + ensureCanRead(name); + return directIODelegate.openInput(name, context); + } else if (useDelegate(name, context)) { // we need to do these checks on the outer directory since the inner doesn't know about pending deletes ensureOpen(); ensureCanRead(name); @@ -159,18 +163,6 @@ public IndexInput openInput(String name, IOContext context) throws IOException { } } - @Override - public IndexInput openInputDirect(String name, IOContext context) throws IOException { - if (directIODelegate == null) { - return openInput(name, context); - } - // we need to do these checks on the outer directory since the inner doesn't know about pending deletes - ensureOpen(); - ensureCanRead(name); - - return directIODelegate.openInput(name, context); - } - @Override public void close() throws IOException { IOUtils.close(super::close, delegate); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java index 4711043fff281..e54fe892ecaab 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java @@ -9,7 +9,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.MergeInfo; import org.apache.lucene.store.ReadAdvice; import org.elasticsearch.blobcache.BlobCacheUtils; import org.elasticsearch.blobcache.common.ByteRange; @@ -21,6 +23,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; import java.util.function.Supplier; @@ -36,7 +40,42 @@ public class CachedBlobContainerIndexInput extends MetadataCachingIndexInput { * a complete part of the {@link #fileInfo} at once in the cache and should not be * used for anything else than what the {@link #prefetchPart(int, Supplier)} method does. */ - public static final IOContext CACHE_WARMING_CONTEXT = new IOContext(IOContext.Context.DEFAULT, null, null, ReadAdvice.NORMAL); + public static final IOContext CACHE_WARMING_CONTEXT = new IOContext() { + @Override + public Context context() { + return Context.DEFAULT; + } + + @Override + public MergeInfo mergeInfo() { + return null; + } + + @Override + public FlushInfo flushInfo() { + return null; + } + + @Override + public Set hints() { + return Set.of(); + } + + @Override + public IOContext withHints(FileOpenHint... hints) { + return this; + } + + @Override + public Optional readAdvice() { + return Optional.of(ReadAdvice.NORMAL); + } + + @Override + public IOContext withReadAdvice(ReadAdvice advice) { + return this; + } + }; private static final Logger logger = LogManager.getLogger(CachedBlobContainerIndexInput.class);