rebase on main

Author: MarkWolters

.github/workflows/unit-tests.yaml

@@ -83,6 +83,13 @@ jobs:
           paths: |
             **/target/surefire-reports/TEST-*.xml
 
+      # Upload Surefire test results as an artifact
+      - name: Upload Surefire Test Results
+        uses: actions/upload-artifact@v4
+        if: always() # ensures upload happens even if tests fail
+        with:
+          name: surefire-results--${{ matrix.isa }}-${{ matrix.jdk }}
+          path: "**/target/surefire-reports/**"
 
   build:
     concurrency:
@@ -120,5 +127,14 @@ jobs:
         with:
           paths: |
             **/target/surefire-reports/TEST-*.xml
+      # Upload Surefire test results as an artifact
+      - name: Upload Surefire Test Results
+        uses: actions/upload-artifact@v4
+        if: always() # ensures upload happens even if tests fail
+        with:
+          name: surefire-results--${{ matrix.os }}-${{ matrix.jdk }}
+          path: "**/target/surefire-reports/**"
+
+
 
 

.gitignore

@@ -2,6 +2,7 @@ target/
 local/
 .mvn/wrapper/maven-wrapper.jar
 .java-version
+.bob/
 
 ### Bench caches
 pq_cache/
@@ -14,6 +15,12 @@ replay_pid*
 .idea
 *.iml
 
+### Eclipse/JDTLS ###
+.settings/
+.classpath
+.project
+.factorypath
+
 ### VS Code ###
 .vscode/
 

benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/IndexConstructionWithStaticSetBenchmark.java

@@ -15,11 +15,9 @@
  */
 package io.github.jbellis.jvector.bench;
 
-import io.github.jbellis.jvector.example.SiftSmall;
 import io.github.jbellis.jvector.example.util.SiftLoader;
 import io.github.jbellis.jvector.graph.*;
 import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider;
-import io.github.jbellis.jvector.util.Bits;
 import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
 import io.github.jbellis.jvector.vector.types.VectorFloat;
 import org.openjdk.jmh.annotations.*;

benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/PQDistanceCalculationBenchmark.java

@@ -130,6 +130,21 @@ public void diversityCalculation(Blackhole blackhole) {
         blackhole.consume(totalSimilarity);
     }
 
+    @Benchmark
+    public void diversityCalculationScoreProvider(Blackhole blackhole) {
+        float totalSimilarity = 0;
+
+        for (int q = 0; q < queryCount; q++) {
+            for (int i = 0; i < vectorCount; i++) {
+                final ScoreFunction sf = buildScoreProvider.diversityScoreFunctionFor(i);
+                float similarity = sf.similarityTo(q);
+                totalSimilarity += similarity;
+            }
+        }
+
+        blackhole.consume(totalSimilarity);
+    }
+
     private VectorFloat<?> createRandomVector(int dimension) {
         VectorFloat<?> vector = VECTOR_TYPE_SUPPORT.createFloatVector(dimension);
         for (int i = 0; i < dimension; i++) {

benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/StaticSetVectorsBenchmark.java

@@ -15,7 +15,7 @@
  */
 package io.github.jbellis.jvector.bench;
 
-import io.github.jbellis.jvector.example.SiftSmall;
+import io.github.jbellis.jvector.example.benchmarks.datasets.SiftSmall;
 import io.github.jbellis.jvector.example.util.SiftLoader;
 import io.github.jbellis.jvector.graph.*;
 import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider;

jvector-base/src/main/java/io/github/jbellis/jvector/disk/ByteBufferIndexWriter.java

@@ -233,6 +233,11 @@ public void writeFloat(float v) {
         buffer.putFloat(v);
     }
 
+    @Override
+    public void writeFloats(float[] floats, int offset, int count) throws IOException {
+        buffer.asFloatBuffer().put(floats, offset, count);
+    }
+
     @Override
     public void writeDouble(double v) {
         buffer.putDouble(v);
@@ -264,7 +269,7 @@ public void writeUTF(String s) throws IOException {
         if (utflen > 65535) {
             throw new IOException("encoded string too long: " + utflen + " bytes");
         }
-        
+
         buffer.putShort((short) utflen);
         buffer.put(bytes);
     }

jvector-base/src/main/java/io/github/jbellis/jvector/disk/IndexWriter.java

@@ -19,6 +19,8 @@
 import java.io.Closeable;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 
 /**
  * Interface for writing index data.
@@ -30,4 +32,12 @@ public interface IndexWriter extends DataOutput, Closeable {
      * @throws IOException if an I/O error occurs
      */
     long position() throws IOException;
+
+    default void writeFloats(float[] floats, int offset, int count) throws IOException {
+        ByteBuffer bb = ByteBuffer.allocate(count * Float.BYTES);
+        // DataOutput specifies BIG_ENDIAN for float
+        bb.order(ByteOrder.BIG_ENDIAN).asFloatBuffer().put(floats, offset, count);
+        bb.rewind();
+        write(bb.array());
+    }
 }

jvector-base/src/main/java/io/github/jbellis/jvector/disk/SimpleMappedReader.java

@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.lang.reflect.Field;
+import java.nio.ByteOrder;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Path;
@@ -73,6 +74,7 @@ public Supplier(Path path) throws IOException {
                     throw new RuntimeException("SimpleMappedReader doesn't support files above 2GB");
                 }
                 this.buffer = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, raf.length());
+                this.buffer.order(ByteOrder.BIG_ENDIAN);
                 this.buffer.load();
             }
         }

jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/OnDiskGraphIndex.java

@@ -402,11 +402,20 @@ public long ramBytesUsed() {
         List<Int2ObjectHashMap<int[]>> inMemoryNeighborsLocal = inMemoryNeighbors.get();
 
         long inMemoryNeighborsBytes = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
-        for (Int2ObjectHashMap<int[]> neighbors : inMemoryNeighborsLocal) {
-            inMemoryNeighborsBytes += neighbors.values().stream().mapToLong(is -> Integer.BYTES * (long) is.length).sum();
-            inMemoryNeighborsBytes += RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+        if (inMemoryNeighborsLocal != null) {
+            for (Int2ObjectHashMap<int[]> neighbors : inMemoryNeighborsLocal) {
+                if (neighbors != null) {
+                    inMemoryNeighborsBytes += neighbors.values().stream().mapToLong(is -> Integer.BYTES * (long) is.length).sum();
+                }
+                inMemoryNeighborsBytes += RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+            }
+        }
+
+        Int2ObjectHashMap<FusedFeature.InlineSource> inMemoryFeaturesLocal  = inMemoryFeatures.get();
+        long inMemoryFeaturesBytes = 0;
+        if (inMemoryFeaturesLocal != null) {
+            inMemoryFeaturesBytes = inMemoryFeaturesLocal.values().stream().mapToLong(is -> Integer.BYTES * is.ramBytesUsed()).sum();
         }
-        long inMemoryFeaturesBytes = inMemoryFeatures.get().values().stream().mapToLong(is -> Integer.BYTES * is.ramBytesUsed()).sum();
         inMemoryFeaturesBytes += RamUsageEstimator.NUM_BYTES_OBJECT_REF;
 
         return Long.BYTES + 6 * Integer.BYTES + RamUsageEstimator.NUM_BYTES_OBJECT_REF

jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/feature/Feature.java

@@ -16,7 +16,8 @@
 
 package io.github.jbellis.jvector.graph.disk.feature;
 
-import java.io.DataOutput;
+import io.github.jbellis.jvector.disk.IndexWriter;
+
 import java.io.IOException;
 import java.util.EnumMap;
 import java.util.function.IntFunction;
@@ -35,9 +36,9 @@ default boolean isFused() {
 
     int featureSize();
 
-    void writeHeader(DataOutput out) throws IOException;
+    void writeHeader(IndexWriter out) throws IOException;
 
-    default void writeInline(DataOutput out, State state) throws IOException {
+    default void writeInline(IndexWriter out, State state) throws IOException {
         // default no-op
     }