Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
target/
local/
.mvn/wrapper/maven-wrapper.jar
.java-version

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ public class BufferedRandomAccessWriter implements RandomAccessWriter {
private final RandomAccessFile raf;
private final DataOutputStream stream;

/**
* Creates a buffered random access writer for the given path.
* @param path the path to write to
* @throws FileNotFoundException if the file cannot be created
*/
public BufferedRandomAccessWriter(Path path) throws FileNotFoundException {
raf = new RandomAccessFile(path.toFile(), "rw");
stream = new DataOutputStream(new BufferedOutputStream(new RandomAccessOutputStream(raf)));
Expand Down Expand Up @@ -88,10 +93,9 @@ public void flush() throws IOException {
}

/**
* return the CRC32 checksum for the range [startOffset .. endOffset)
* <p>
* the file pointer will be left at endOffset.
* <p>
* Returns the CRC32 checksum for the range [startOffset .. endOffset)
*
* The file pointer will be left at endOffset.
*/
@Override
public long checksum(long startOffset, long endOffset) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,13 @@
* RandomAccessReader that reads from a ByteBuffer
*/
public class ByteBufferReader implements RandomAccessReader {
/** The underlying ByteBuffer. */
protected final ByteBuffer bb;

/**
* Creates a ByteBufferReader from the given ByteBuffer.
* @param sourceBB the source ByteBuffer
*/
public ByteBufferReader(ByteBuffer sourceBB) {
bb = sourceBB;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,14 @@
import java.io.DataOutput;
import java.io.IOException;

/**
* Interface for writing index data.
*/
public interface IndexWriter extends DataOutput, Closeable {
/**
* Returns the current position in the output.
* @return the current position in the output
* @throws IOException if an I/O error occurs
*/
long position() throws IOException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,32 +32,99 @@
* uses the ReaderSupplier API to create a RandomAccessReader per thread, as needed.
*/
public interface RandomAccessReader extends AutoCloseable {
/**
* Seeks to the specified offset.
* @param offset the offset to seek to
* @throws IOException if an I/O error occurs
*/
void seek(long offset) throws IOException;

/**
* Returns the current position.
* @return the current position
* @throws IOException if an I/O error occurs
*/
long getPosition() throws IOException;

/**
* Reads an integer.
* @return the integer value
* @throws IOException if an I/O error occurs
*/
int readInt() throws IOException;

/**
* Reads a float.
* @return the float value
* @throws IOException if an I/O error occurs
*/
float readFloat() throws IOException;

/**
* Reads a long.
* @return the long value
* @throws IOException if an I/O error occurs
*/
long readLong() throws IOException;

/**
* Reads bytes into the array.
* @param bytes the byte array to read into
* @throws IOException if an I/O error occurs
*/
void readFully(byte[] bytes) throws IOException;

/**
* Reads bytes into the buffer.
* @param buffer the ByteBuffer to read into
* @throws IOException if an I/O error occurs
*/
void readFully(ByteBuffer buffer) throws IOException;

/**
* Reads floats into the array.
* @param floats the float array to read into
* @throws IOException if an I/O error occurs
*/
default void readFully(float[] floats) throws IOException {
read(floats, 0, floats.length);
}

/**
* Reads longs into the array.
* @param vector the long array to read into
* @throws IOException if an I/O error occurs
*/
void readFully(long[] vector) throws IOException;

/**
* Reads integers into the array.
* @param ints the int array to read into
* @param offset the offset in the array
* @param count the number of integers to read
* @throws IOException if an I/O error occurs
*/
void read(int[] ints, int offset, int count) throws IOException;

/**
* Reads floats into the array.
* @param floats the float array to read into
* @param offset the offset in the array
* @param count the number of floats to read
* @throws IOException if an I/O error occurs
*/
void read(float[] floats, int offset, int count) throws IOException;

/**
* Closes this reader.
* @throws IOException if an I/O error occurs
*/
void close() throws IOException;

// Length of the reader slice
/**
* Returns the length of the reader slice.
* @return the length
* @throws IOException if an I/O error occurs
*/
long length() throws IOException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,25 @@
* A DataOutput that adds methods for random access writes
*/
public interface RandomAccessWriter extends IndexWriter {
/**
* Seeks to the specified position.
* @param position the position to seek to
* @throws IOException if an I/O error occurs
*/
void seek(long position) throws IOException;

/**
* Flushes any buffered data.
* @throws IOException if an I/O error occurs
*/
void flush() throws IOException;

/**
* Computes a checksum for the specified range.
* @param startOffset the start offset
* @param endOffset the end offset
* @return the checksum value
* @throws IOException if an I/O error occurs
*/
long checksum(long startOffset, long endOffset) throws IOException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@
*/
public interface ReaderSupplier extends AutoCloseable {
/**
* Creates a new reader.
* @return a new reader. It is up to the caller to re-use these readers or close them,
* the ReaderSupplier is not responsible for caching them.
* @throws IOException if an I/O error occurs
*/
RandomAccessReader get() throws IOException;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,26 @@
import java.util.logging.Level;
import java.util.logging.Logger;

/**
* Factory for creating ReaderSupplier instances.
*/
public class ReaderSupplierFactory {
private static final Logger LOG = Logger.getLogger(ReaderSupplierFactory.class.getName());
private static final String MEMORY_SEGMENT_READER_CLASSNAME = "io.github.jbellis.jvector.disk.MemorySegmentReader$Supplier";
private static final String MMAP_READER_CLASSNAME = "io.github.jbellis.jvector.example.util.MMapReader$Supplier";

/**
* Private constructor to prevent instantiation.
*/
private ReaderSupplierFactory() {
}

/**
* Opens a ReaderSupplier for the given path.
* @param path the path to open
* @return a ReaderSupplier for the path
* @throws IOException if an I/O error occurs
*/
public static ReaderSupplier open(Path path) throws IOException {
try {
// prefer MemorySegmentReader (available under JDK 20+)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,18 @@ public void close() {
// Individual readers don't close anything
}

/**
* Supplier for SimpleMappedReader instances.
*/
public static class Supplier implements ReaderSupplier {
private final MappedByteBuffer buffer;
private static final Unsafe unsafe = getUnsafe();

/**
* Creates a Supplier for the given path.
* @param path the path to read from
* @throws IOException if an I/O error occurs
*/
public Supplier(Path path) throws IOException {
try (var raf = new RandomAccessFile(path.toString(), "r")) {
if (raf.length() > Integer.MAX_VALUE) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,17 @@
import java.nio.ByteOrder;
import java.nio.file.Path;

// TODO what are the low-hanging optimization options here?
// The requirement that we need to read from a file that is potentially changing in length
// limits our options.
/**
* Simple implementation of RandomAccessReader using RandomAccessFile.
*/
public class SimpleReader implements RandomAccessReader {
RandomAccessFile raf;

/**
* Creates a SimpleReader for the given path.
* @param path the path to read from
* @throws FileNotFoundException if the file cannot be found
*/
public SimpleReader(Path path) throws FileNotFoundException {
raf = new RandomAccessFile(path.toFile(), "r");
}
Expand Down Expand Up @@ -105,9 +110,16 @@ public long length() throws IOException {
return raf.length();
}

/**
* Supplier for SimpleReader instances.
*/
public static class Supplier implements ReaderSupplier {
private final Path path;

/**
* Creates a Supplier for the given path.
* @param path the path to read from
*/
public Supplier(Path path) {
this.path = path;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ public class SimpleWriter implements IndexWriter {
private final FileOutputStream fos;
private final DataOutputStream dos;

/**
* Creates a SimpleWriter for the given path.
* @param path the path to write to
* @throws IOException if an I/O error occurs
*/
public SimpleWriter(Path path) throws IOException {
fos = new FileOutputStream(path.toFile());
dos = new DataOutputStream(fos);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,14 @@

package io.github.jbellis.jvector.exceptions;

/**
* RuntimeException wrapper for InterruptedException.
*/
public final class ThreadInterruptedException extends RuntimeException {
/**
* Creates a ThreadInterruptedException wrapping the given InterruptedException.
* @param ie the InterruptedException to wrap
*/
public ThreadInterruptedException(InterruptedException ie) {
super(ie);
}
Expand Down