Better control over setting string formats for the benchmark metrics (#461)

marianotepper · web-flow · commit e574a9dccf5b · 2025-05-05T19:32:11.000-07:00
diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java
@@ -377,10 +377,10 @@ private static void testConfiguration(ConfiguredSystem cs,
         System.out.format("Using %s:%n", cs.index);
         // 1) Select benchmarks to run
         List<QueryBenchmark> benchmarks = List.of(
-                new ThroughputBenchmark(2, 0.1),
-                new LatencyBenchmark(),
-                new CountBenchmark(),
-                new AccuracyBenchmark()
+                ThroughputBenchmark.createDefault(2, 0.1),
+                LatencyBenchmark.createDefault(),
+                CountBenchmark.createDefault(),
+                AccuracyBenchmark.createDefault()
         );
         QueryTester tester = new QueryTester(benchmarks);
 
diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/AccuracyBenchmark.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/AccuracyBenchmark.java
@@ -29,33 +29,46 @@
  * Measures average recall and/or the mean average precision.
  */
 public class AccuracyBenchmark extends AbstractQueryBenchmark {
-    static private final String DEFAULT_FORMAT = ".2f";
+    private static final String DEFAULT_FORMAT = ".2f";
 
-    private final boolean computeRecall;
-    private final boolean computeMAP;
-    private final String formatRecall;
-    private final String formatMAP;
+    private boolean computeRecall;
+    private boolean computeMAP;
+    private String formatRecall;
+    private String formatMAP;
 
-    public AccuracyBenchmark(boolean computeRecall, boolean computeMAP, String formatRecall, String formatMAP) {
-        if (!(computeRecall || computeMAP)) {
-            throw new IllegalArgumentException("At least one parameter must be set to true");
-        }
+    public static AccuracyBenchmark createDefault() {
+        return new AccuracyBenchmark(true, false, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    }
+
+    public static AccuracyBenchmark createEmpty() {
+        return new AccuracyBenchmark(false, false, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    }
+
+    private AccuracyBenchmark(boolean computeRecall, boolean computeMAP, String formatRecall, String formatMAP) {
         this.computeRecall = computeRecall;
         this.computeMAP = computeMAP;
         this.formatRecall = formatRecall;
         this.formatMAP = formatMAP;
     }
 
-    public AccuracyBenchmark() {
-        this(true, false, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    public AccuracyBenchmark displayRecall() {
+        return displayRecall(DEFAULT_FORMAT);
     }
 
-    public AccuracyBenchmark(String formatRecall) {
-        this(true, false, formatRecall, DEFAULT_FORMAT);
+    public AccuracyBenchmark displayRecall(String format) {
+        this.computeRecall = true;
+        this.formatRecall = format;
+        return this;
     }
 
-    public AccuracyBenchmark(String formatRecall, String formatMAP) {
-        this(true, true, formatRecall, formatMAP);
+    public AccuracyBenchmark displayMAP() {
+        return displayMAP(DEFAULT_FORMAT);
+    }
+
+    public AccuracyBenchmark displayMAP(String format) {
+        this.computeMAP = true;
+        this.formatMAP = format;
+        return this;
     }
 
     @Override
@@ -71,6 +84,10 @@ public List<Metric> runBenchmark(
             boolean usePruning,
             int queryRuns) {
 
+        if (!(computeRecall || computeMAP)) {
+            throw new RuntimeException("At least one metric must be displayed");
+        }
+
         int totalQueries = cs.getDataSet().queryVectors.size();
 
         // execute all queries in parallel and collect results
diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/CountBenchmark.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/CountBenchmark.java
@@ -23,26 +23,30 @@
 
 import io.github.jbellis.jvector.example.Grid.ConfiguredSystem;
 import io.github.jbellis.jvector.graph.SearchResult;
-import org.apache.commons.math3.analysis.function.Abs;
 
 /**
  * Measures average node‐visit and node‐expand counts over N runs.
  */
 public class CountBenchmark extends AbstractQueryBenchmark {
-    static private final String DEFAULT_FORMAT = ".1f";
+    private static final String DEFAULT_FORMAT = ".1f";
 
-    private final boolean computeAvgNodesVisited;
-    private final boolean computeAvgNodesExpanded;
-    private final boolean computeAvgNodesExpandedBaseLayer;
-    private final String formatAvgNodesVisited;
-    private final String formatAvgNodesExpanded;
-    private final String formatAvgNodesExpandedBaseLayer;
+    private boolean computeAvgNodesVisited;
+    private boolean computeAvgNodesExpanded;
+    private boolean computeAvgNodesExpandedBaseLayer;
+    private String formatAvgNodesVisited;
+    private String formatAvgNodesExpanded;
+    private String formatAvgNodesExpandedBaseLayer;
 
-    public CountBenchmark(boolean computeAvgNodesVisited, boolean computeAvgNodesExpanded, boolean computeAvgNodesExpandedBaseLayer,
+    public static CountBenchmark createDefault() {
+        return new CountBenchmark(true, false, false, DEFAULT_FORMAT, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    }
+
+    public static CountBenchmark createEmpty() {
+        return new CountBenchmark(false, false, false, DEFAULT_FORMAT, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    }
+
+    private CountBenchmark(boolean computeAvgNodesVisited, boolean computeAvgNodesExpanded, boolean computeAvgNodesExpandedBaseLayer,
                           String formatAvgNodesVisited, String formatAvgNodesExpanded, String formatAvgNodesExpandedBaseLayer) {
-        if (!(computeAvgNodesVisited || computeAvgNodesExpanded || computeAvgNodesExpandedBaseLayer)) {
-            throw new IllegalArgumentException("At least one parameter must be set to true");
-        }
         this.computeAvgNodesVisited = computeAvgNodesVisited;
         this.computeAvgNodesExpanded = computeAvgNodesExpanded;
         this.computeAvgNodesExpandedBaseLayer = computeAvgNodesExpandedBaseLayer;
@@ -51,12 +55,34 @@ public CountBenchmark(boolean computeAvgNodesVisited, boolean computeAvgNodesExp
         this.formatAvgNodesExpandedBaseLayer = formatAvgNodesExpandedBaseLayer;
     }
 
-    public CountBenchmark() {
-        this(true, false, false, DEFAULT_FORMAT, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    public CountBenchmark displayAvgNodesVisited() {
+        return displayAvgNodesVisited(DEFAULT_FORMAT);
+    }
+
+    public CountBenchmark displayAvgNodesVisited(String format) {
+        this.computeAvgNodesVisited = true;
+        this.formatAvgNodesVisited = format;
+        return this;
     }
 
-    public CountBenchmark(String formatAvgNodesVisited, String formatAvgNodesExpanded, String formatAvgNodesExpandedBaseLayer) {
-        this(true, true, true, formatAvgNodesVisited, formatAvgNodesExpanded, formatAvgNodesExpandedBaseLayer);
+    public CountBenchmark displayAvgNodesExpanded() {
+        return displayAvgNodesExpanded(DEFAULT_FORMAT);
+    }
+
+    public CountBenchmark displayAvgNodesExpanded(String format) {
+        this.computeAvgNodesExpanded = true;
+        this.formatAvgNodesExpanded = format;
+        return this;
+    }
+
+    public CountBenchmark displayAvgNodesExpandedBaseLayer() {
+        return displayAvgNodesExpandedBaseLayer(DEFAULT_FORMAT);
+    }
+
+    public CountBenchmark displayAvgNodesExpandedBaseLayer(String format) {
+        this.computeAvgNodesExpandedBaseLayer = true;
+        this.formatAvgNodesExpandedBaseLayer = format;
+        return this;
     }
 
     @Override
@@ -72,6 +98,10 @@ public List<Metric> runBenchmark(
             boolean usePruning,
             int queryRuns) {
 
+        if (!(computeAvgNodesVisited || computeAvgNodesExpanded || computeAvgNodesExpandedBaseLayer)) {
+            throw new RuntimeException("At least one metric must be displayed");
+        }
+
         LongAdder nodesVisited = new LongAdder();
         LongAdder nodesExpanded = new LongAdder();
         LongAdder nodesExpandedBaseLayer = new LongAdder();
diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ExecutionTimeBenchmark.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ExecutionTimeBenchmark.java
@@ -25,17 +25,22 @@
  * Measures average execution time over N runs through all queries in parallel.
  */
 public class ExecutionTimeBenchmark extends AbstractQueryBenchmark {
-    static private final String DEFAULT_FORMAT = ".1f";
+    private static final String DEFAULT_FORMAT = ".1f";
 
     private static volatile long SINK;
-    private final String format;
+    private String format;
 
-    public ExecutionTimeBenchmark(String format) {
+    public static ExecutionTimeBenchmark createDefault() {
+        return new ExecutionTimeBenchmark(DEFAULT_FORMAT);
+    }
+
+    private ExecutionTimeBenchmark(String format) {
         this.format = format;
     }
 
-    public ExecutionTimeBenchmark() {
-        this.format = DEFAULT_FORMAT;
+    public ExecutionTimeBenchmark setFormat(String format) {
+        this.format = format;
+        return this;
     }
 
     @Override
diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/LatencyBenchmark.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/LatencyBenchmark.java
@@ -28,22 +28,27 @@
  * and counts correct top‐K results.
  */
 public class LatencyBenchmark extends AbstractQueryBenchmark {
-    static private final String DEFAULT_FORMAT = ".3f";
+    private static final String DEFAULT_FORMAT = ".3f";
 
-    private final boolean computeAvgLatency;
-    private final boolean computeLatencySTD;
-    private final boolean computeP999Latency;
-    private final String formatAvgLatency;
-    private final String formatLatencySTD;
-    private final String formatP999Latency;
+    private boolean computeAvgLatency;
+    private boolean computeLatencySTD;
+    private boolean computeP999Latency;
+    private String formatAvgLatency;
+    private String formatLatencySTD;
+    private String formatP999Latency;
 
     private static volatile long SINK;
 
-    public LatencyBenchmark(boolean computeAvgLatency, boolean computeLatencySTD, boolean computeP999Latency,
+    public static LatencyBenchmark createDefault() {
+        return new LatencyBenchmark(true, false, false, DEFAULT_FORMAT, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    }
+
+    public static LatencyBenchmark createEmpty() {
+        return new LatencyBenchmark(false, false, false, DEFAULT_FORMAT, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    }
+
+    private LatencyBenchmark(boolean computeAvgLatency, boolean computeLatencySTD, boolean computeP999Latency,
                             String formatAvgLatency, String formatLatencySTD, String formatP999Latency) {
-        if (!(computeAvgLatency || computeLatencySTD || computeP999Latency)) {
-            throw new IllegalArgumentException("At least one parameter must be set to true");
-        }
         this.computeAvgLatency = computeAvgLatency;
         this.computeLatencySTD = computeLatencySTD;
         this.computeP999Latency = computeP999Latency;
@@ -52,12 +57,34 @@ public LatencyBenchmark(boolean computeAvgLatency, boolean computeLatencySTD, bo
         this.formatP999Latency = formatP999Latency;
     }
 
-    public LatencyBenchmark() {
-        this(true, false, false, DEFAULT_FORMAT, DEFAULT_FORMAT, DEFAULT_FORMAT);
+    public LatencyBenchmark displayAvgLatency() {
+        return displayAvgLatency(DEFAULT_FORMAT);
     }
 
-    public LatencyBenchmark(String formatAvgLatency, String formatLatencySTD, String formatP999Latency) {
-        this(true, true, true, formatAvgLatency, formatLatencySTD, formatP999Latency);
+    public LatencyBenchmark displayAvgLatency(String format) {
+        this.computeAvgLatency = true;
+        this.formatAvgLatency = format;
+        return this;
+    }
+
+    public LatencyBenchmark displayLatencySTD() {
+        return displayLatencySTD(DEFAULT_FORMAT);
+    }
+
+    public LatencyBenchmark displayLatencySTD(String format) {
+        this.computeLatencySTD = true;
+        this.formatLatencySTD = format;
+        return this;
+    }
+
+    public LatencyBenchmark displayP999Latency() {
+        return displayP999Latency(DEFAULT_FORMAT);
+    }
+
+    public LatencyBenchmark displayP999Latency(String format) {
+        this.computeP999Latency = true;
+        this.formatP999Latency = format;
+        return this;
     }
 
     @Override
@@ -73,6 +100,10 @@ public List<Metric> runBenchmark(
             boolean usePruning,
             int queryRuns) {
 
+        if (!(computeAvgLatency || computeLatencySTD || computeP999Latency)) {
+            throw new IllegalArgumentException("At least one parameter must be set to true");
+        }
+
         int totalQueries = cs.getDataSet().queryVectors.size();
         double mean = 0.0;
         double m2 = 0.0;
diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java
@@ -27,22 +27,27 @@
  * Measures throughput (queries/sec) with an optional warmup phase.
  */
 public class ThroughputBenchmark extends AbstractQueryBenchmark {
-    static private final String DEFAULT_FORMAT = ".1f";
+    private static final String DEFAULT_FORMAT = ".1f";
 
     private static volatile long SINK;
 
     private final int warmupRuns;
     private final double warmupRatio;
-    private final String format;
+    private String format;
 
-    public ThroughputBenchmark(int warmupRuns, double warmupRatio, String format) {
+    public static ThroughputBenchmark createDefault(int warmupRuns, double warmupRatio) {
+        return new ThroughputBenchmark(warmupRuns, warmupRatio, DEFAULT_FORMAT);
+    }
+
+    private ThroughputBenchmark(int warmupRuns, double warmupRatio, String format) {
         this.warmupRuns = warmupRuns;
         this.warmupRatio = warmupRatio;
         this.format = format;
     }
 
-    public ThroughputBenchmark(int warmupRuns, double warmupRatio) {
-        this(warmupRuns, warmupRatio, DEFAULT_FORMAT);
+    public ThroughputBenchmark setFormat(String format) {
+        this.format = format;
+        return this;
     }
 
     @Override
