Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ private static Version parseUnchecked(String version) {
public static final IndexVersion UPGRADE_TO_LUCENE_10_2_1 = def(9_023_00_0, Version.LUCENE_10_2_1);
public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = def(9_024_0_00, Version.LUCENE_10_2_1);
public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_025_0_00, Version.LUCENE_10_2_1);

public static final IndexVersion UPGRADE_TO_LUCENE_10_3_0 = def(9_050_00_0, Version.LUCENE_10_3_0);
/*
* STOP! READ THIS FIRST! No, really,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,7 @@
package org.elasticsearch.index.codec.vectors.es818;

import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;

import java.io.IOException;

/**
* A hook for {@link DirectIOLucene99FlatVectorsReader} to specify the input should be opened using DirectIO.
* Remove when IOContext allows more extensible payloads to be specified.
*/
public interface DirectIOIndexInputSupplier {
IndexInput openInputDirect(String name, IOContext context) throws IOException;
public enum DirectIOHint implements IOContext.FileOpenHint {
INSTANCE
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,14 @@
import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.store.ReadAdvice;

import java.io.IOException;
import java.util.Optional;
import java.util.Set;

/**
* Copied from Lucene99FlatVectorsFormat in Lucene 10.1
Expand Down Expand Up @@ -61,22 +67,62 @@ public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOExceptio
return new Lucene99FlatVectorsWriter(state, vectorsScorer);
}

private static final IOContext DIRECT_IO_CONTEXT = new IOContext() {
@Override
public Context context() {
return Context.DEFAULT;
}

@Override
public MergeInfo mergeInfo() {
return null;
}

@Override
public FlushInfo flushInfo() {
return null;
}

@Override
public Set<FileOpenHint> hints() {
return Set.of(DirectIOHint.INSTANCE);
}

@Override
public IOContext withHints(FileOpenHint... hints) {
return this;
}

@Override
public Optional<ReadAdvice> readAdvice() {
return Optional.empty();
}

@Override
public IOContext withReadAdvice(ReadAdvice advice) {
return this;
}
};

@Override
public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {
if (DirectIOLucene99FlatVectorsReader.shouldUseDirectIO(state)) {
// Use mmap for merges and direct I/O for searches.
// TODO: Open the mmap file with sequential access instead of random (current behavior).
return new MergeReaderWrapper(
new DirectIOLucene99FlatVectorsReader(state, vectorsScorer),
new Lucene99FlatVectorsReader(state, vectorsScorer)
);
} else {
return new Lucene99FlatVectorsReader(state, vectorsScorer);
}
SegmentReadState directIOState = new SegmentReadState(
state.directory,
state.segmentInfo,
state.fieldInfos,
DIRECT_IO_CONTEXT,
state.segmentSuffix
);
// Use mmap for merges and direct I/O for searches.
// TODO: Open the mmap file with sequential access instead of random (current behavior).
return new MergeReaderWrapper(
new Lucene99FlatVectorsReader(directIOState, vectorsScorer),
new Lucene99FlatVectorsReader(state, vectorsScorer)
);
}

@Override
public String toString() {
return "ES818FlatVectorsFormat(" + "vectorsScorer=" + vectorsScorer + ')';
return "Lucene99FlatVectorsFormat(" + "vectorsScorer=" + vectorsScorer + ')';
}
}
Loading