update to create csv and script to create viz

MarkWolters · MarkWolters · commit 849fe70f9192 · 2025-07-10T15:16:19.000-04:00
diff --git a/.github/workflows/run-bench.yml b/.github/workflows/run-bench.yml
@@ -4,11 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       benchmark_config:
-        description: 'Benchmark configuration file (leave empty for default)'
-        required: false
-        default: ''
-      jdk_version:
-        description: 'Override JDK version (leave empty to use matrix)'
+        description: 'Benchmark dataset regex (leave empty for all)'
         required: false
         default: ''
   push:
@@ -64,7 +60,7 @@ jobs:
           # Use the jar-with-dependencies which includes all required dependencies
           java ${{ matrix.jdk >= 20 && '--enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector' || '' }} \
             ${{ matrix.jdk >= 22 && '-Djvector.experimental.enable_native_vectorization=true' || '' }} \
-            -cp jvector-examples/target/jvector-examples-*-jar-with-dependencies.jar io.github.jbellis.jvector.example.AutoBenchYAML --output bench-results.json ${{ inputs.benchmark_config != '' && inputs.benchmark_config || 'jvector-examples/yaml-configs/default.yml' }}
+            -cp jvector-examples/target/jvector-examples-*-jar-with-dependencies.jar io.github.jbellis.jvector.example.AutoBenchYAML --output bench-results 
           
           # List files in current directory to help with debugging
           echo "Files in current directory:"
@@ -76,37 +72,37 @@ jobs:
           name: bench-results-${{ matrix.isa }}-jdk${{ matrix.jdk }}
           path: |
             bench-results.json
-            bench-results.log
+            bench-results.csv
           if-no-files-found: warn
 
-      - name: Download Previous Benchmark Results
-        uses: dawidd6/action-download-artifact@v2
-        continue-on-error: true
-        with:
-          workflow: run-bench.yml
-          name: bench-results-${{ matrix.isa }}-jdk${{ matrix.jdk }}
-          path: previous-results
-          skip_unpack: false
-          if_no_artifact_found: warn
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-
-      - name: Install Python Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install argparse
-
-      - name: Compare Benchmark Results
-        if: success() && hashFiles('previous-results/bench-results.json') != ''
-        run: |
-          python compare_benchmarks.py bench-results.json previous-results/bench-results.json --output benchmark-comparison.md
-
-      - name: Upload Comparison Report
-        if: success() && hashFiles('benchmark-comparison.md') != ''
-        uses: actions/upload-artifact@v4
-        with:
-          name: benchmark-comparison-${{ matrix.isa }}-jdk${{ matrix.jdk }}
-          path: benchmark-comparison.md
+#      - name: Download Previous Benchmark Results
+#        uses: dawidd6/action-download-artifact@v2
+#        continue-on-error: true
+#        with:
+#          workflow: run-bench.yml
+#          name: bench-results-${{ matrix.isa }}-jdk${{ matrix.jdk }}
+#          path: previous-results
+#          skip_unpack: false
+#          if_no_artifact_found: warn
+#
+#      - name: Set up Python
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version: '3.x'
+#
+#      - name: Install Python Dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install argparse
+#
+#      - name: Compare Benchmark Results
+#        if: success() && hashFiles('previous-results/bench-results.json') != ''
+#        run: |
+#          python compare_benchmarks.py bench-results.json previous-results/bench-results.json --output benchmark-comparison.md
+#
+#      - name: Upload Comparison Report
+#        if: success() && hashFiles('benchmark-comparison.md') != ''
+#        uses: actions/upload-artifact@v4
+#        with:
+#          name: benchmark-comparison-${{ matrix.isa }}-jdk${{ matrix.jdk }}
+#          path: benchmark-comparison.md
diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/AutoBenchYAML.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/AutoBenchYAML.java
@@ -21,17 +21,21 @@
 import io.github.jbellis.jvector.example.util.BenchmarkSummarizer.SummaryStats;
 import io.github.jbellis.jvector.example.util.DataSet;
 import io.github.jbellis.jvector.example.util.DataSetLoader;
+import io.github.jbellis.jvector.example.yaml.ConstructionParameters;
 import io.github.jbellis.jvector.example.yaml.MultiConfig;
+import io.github.jbellis.jvector.example.yaml.SearchParameters;
 import io.github.jbellis.jvector.graph.disk.feature.FeatureId;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -53,7 +57,7 @@ private static List<String> getAllDatasetNames() {
         // neighborhood-watch-100k datasets
 //        allDatasets.add("ada002-100k");
         allDatasets.add("cohere-english-v3-100k");
-//        allDatasets.add("openai-v3-small-100k");
+        allDatasets.add("openai-v3-small-100k");
 //        allDatasets.add("gecko-100k");
 //        allDatasets.add("openai-v3-large-3072-100k");
 //        allDatasets.add("openai-v3-large-1536-100k");
@@ -122,7 +126,8 @@ public static void main(String[] args) throws IOException {
                         datasetName = datasetName.substring(0, datasetName.length() - ".hdf5".length());
                     }
                     
-                    MultiConfig config = MultiConfig.getDefaultConfig(datasetName);
+                    MultiConfig config = MultiConfig.getDefaultConfig("autoDefault");
+                    config.dataset = datasetName;
                     logger.info("Using configuration: {}", config);
 
                     results.addAll(Grid.runAllAndCollectResults(ds, 
@@ -143,50 +148,39 @@ public static void main(String[] args) throws IOException {
             }
         }
 
-        // Process YAML configuration files
-        List<String> configNames = Arrays.stream(filteredArgs).filter(s -> s.endsWith(".yml")).collect(Collectors.toList());
-        if (!configNames.isEmpty()) {
-            for (var configName : configNames) {
-                logger.info("Processing configuration file: {}", configName);
-                
-                try {
-                    MultiConfig config = MultiConfig.getConfig(configName);
-                    String datasetName = config.dataset;
-                    logger.info("Configuration specifies dataset: {}", datasetName);
-
-                    logger.info("Loading dataset: {}", datasetName);
-                    DataSet ds = DataSetLoader.loadDataSet(datasetName);
-                    logger.info("Dataset loaded: {} with {} vectors", datasetName, ds.baseVectors.size());
-
-                    results.addAll(Grid.runAllAndCollectResults(ds, 
-                            config.construction.outDegree, 
-                            config.construction.efConstruction,
-                            config.construction.neighborOverflow, 
-                            config.construction.addHierarchy,
-                            config.construction.getFeatureSets(), 
-                            config.construction.getCompressorParameters(),
-                            config.search.getCompressorParameters(), 
-                            config.search.topKOverquery, 
-                            config.search.useSearchPruning));
-                    
-                    logger.info("Benchmark completed for YAML config: {}", configName);
-                } catch (Exception e) {
-                    logger.error("Exception while processing YAML config {}", configName, e);
-                }
-            }
-        }
-
         // Calculate summary statistics
         try {
             SummaryStats stats = BenchmarkSummarizer.summarize(results);
             logger.info("Benchmark summary: {}", stats.toString());
 
-            // Write results to JSON file
+            // Write results to csv file and details to json
+            File detailsFile = new File(outputPath + ".json");
             ObjectMapper mapper = new ObjectMapper();
-            File outputFile = new File(outputPath);
-            mapper.writerWithDefaultPrettyPrinter().writeValue(outputFile, results);
-            logger.info("Benchmark results written to {} (file exists: {})", outputPath, outputFile.exists());
+            mapper.writerWithDefaultPrettyPrinter().writeValue(detailsFile, results);
+
+            File outputFile = new File(outputPath + ".csv");
             
+            // Get summary statistics by dataset
+            Map<String, SummaryStats> statsByDataset = BenchmarkSummarizer.summarizeByDataset(results);
+            
+            // Write CSV data
+            try (FileWriter writer = new FileWriter(outputFile)) {
+                // Write CSV header
+                writer.write("dataset,QPS,Mean Latency,Recall@10\n");
+                
+                // Write one row per dataset with average metrics
+                for (Map.Entry<String, SummaryStats> entry : statsByDataset.entrySet()) {
+                    String dataset = entry.getKey();
+                    SummaryStats datasetStats = entry.getValue();
+                    
+                    writer.write(dataset + ",");
+                    writer.write(datasetStats.getAvgQps() + ",");
+                    writer.write(datasetStats.getAvgLatency() + ",");
+                    writer.write(datasetStats.getAvgRecall() + "\n");
+                }
+            }
+
+            logger.info("Benchmark results written to {} (file exists: {})", outputPath, outputFile.exists());
             // Double check that the file was created and log its size
             if (outputFile.exists()) {
                 logger.info("Output file size: {} bytes", outputFile.length());
@@ -197,4 +191,5 @@ public static void main(String[] args) throws IOException {
             logger.error("Exception during final processing", e);
         }
     }
+
 }
diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/util/BenchmarkSummarizer.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/util/BenchmarkSummarizer.java
@@ -191,4 +191,35 @@ private static Double convertToDouble(Object value) {
         }
         return null;
     }
+    
+    /**
+     * Calculate summary statistics grouped by dataset from a list of benchmark results
+     * @param results List of benchmark results to summarize
+     * @return Map of dataset names to their summary statistics
+     */
+    public static Map<String, SummaryStats> summarizeByDataset(List<BenchResult> results) {
+        if (results == null || results.isEmpty()) {
+            return Map.of();
+        }
+
+        // Group results by dataset
+        Map<String, List<BenchResult>> resultsByDataset = new java.util.HashMap<>();
+        for (BenchResult result : results) {
+            if (result.dataset == null) continue;
+            
+            resultsByDataset.computeIfAbsent(result.dataset, k -> new java.util.ArrayList<>()).add(result);
+        }
+        
+        // Calculate summary stats for each dataset
+        Map<String, SummaryStats> statsByDataset = new java.util.HashMap<>();
+        for (Map.Entry<String, List<BenchResult>> entry : resultsByDataset.entrySet()) {
+            String dataset = entry.getKey();
+            List<BenchResult> datasetResults = entry.getValue();
+            
+            SummaryStats stats = summarize(datasetResults);
+            statsByDataset.put(dataset, stats);
+        }
+        
+        return statsByDataset;
+    }
 }
diff --git a/jvector-examples/yaml-configs/autoDefault.yml b/jvector-examples/yaml-configs/autoDefault.yml
@@ -0,0 +1,33 @@
+version: 5
+
+dataset: cohere-english-v3-100k
+
+construction:
+  outDegree: [32]
+  efConstruction: [100]
+  neighborOverflow: [1.2f]
+  addHierarchy: [Yes]
+  refineFinalGraph: [Yes]
+  compression:
+    - type: PQ
+      parameters:
+        m: 192 # we can either specify the integer m or the integer mFactor. In this case, m will be set to the data dimensionality divided by mFactor
+        # mFactor: 8
+        # k: 256 # optional parameter. By default, k=256
+        centerData: No
+        anisotropicThreshold: -1.0 # optional parameter. By default, anisotropicThreshold=-1 (i.e., no anisotropy)
+  reranking:
+    - NVQ
+  useSavedIndexIfExists: Yes
+
+search:
+  topKOverquery:
+    10: [1.0]
+  useSearchPruning: [Yes]
+  compression:
+    - type: PQ
+      parameters:
+        m: 192
+        # k: 256 # optional parameter. By default, k=256
+        centerData: No
+        anisotropicThreshold: -1.0 # optional parameter. By default, anisotropicThreshold=-1 (i.e., no anisotropy)
diff --git a/visualize_benchmarks.py b/visualize_benchmarks.py
