separate viz job

Author: MarkWolters

.github/workflows/run-bench.yml

@@ -145,12 +145,8 @@ jobs:
           # Run the benchmark
           echo "Running benchmark for branch ${{ matrix.branch }}"
           java ${{ matrix.jdk >= 20 && '--enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector' || '' }} \
-            -jar jvector-examples/target/jvector-examples-*-jar-with-dependencies.jar \
-            --config jvector-examples/yaml-configs/autoDefault.yml \
-            --output ${{ matrix.branch }}-bench-results || {
-              echo "Warning: Benchmark failed for branch ${{ matrix.branch }}. This may be expected if the branch doesn't have the required code."
-              exit 0
-            }
+            ${{ matrix.jdk >= 22 && '-Djvector.experimental.enable_native_vectorization=true' || '' }} \
+            -cp jvector-examples/target/jvector-examples-*-jar-with-dependencies.jar io.github.jbellis.jvector.example.AutoBenchYAML --output ${{ matrix.branch }}-bench-results
 
           # Move the results to the benchmark_results directory
           mv ${{ matrix.branch }}-bench-results.csv benchmark_results/ || true
@@ -161,12 +157,24 @@ jobs:
       - name: Upload Individual Benchmark Results
         uses: actions/upload-artifact@v4
         with:
-          name: benchmark-results-${{ matrix.isa }}-jdk${{ matrix.jdk }}
+          name: benchmark-results-${{ matrix.isa }}-jdk${{ matrix.jdk }}-${{ matrix.branch }}
           path: |
             benchmark_results/*.csv
             benchmark_results/*.json
           if-no-files-found: warn
 
+  # Job to combine results and create visualizations
+  combine-results:
+    needs: test-avx512
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all benchmark results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: benchmark-results-*
+          path: all-benchmark-results
+          merge-multiple: true
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -186,8 +194,8 @@ jobs:
           import pandas as pd
           import matplotlib.pyplot as plt
 
-          # Find all CSV files in the benchmark_results directory
-          csv_files = glob.glob('benchmark_results/*-bench-results.csv')
+          # Find all CSV files in the all-benchmark-results directory
+          csv_files = glob.glob('all-benchmark-results/**/*-bench-results.csv', recursive=True)
 
           if not csv_files:
               print("No benchmark results found! Checking other possible locations...")
@@ -279,8 +287,6 @@ jobs:
         with:
           name: benchmark-comparison-results
           path: |
-            benchmark_results/*.csv
-            benchmark_results/*.json
             all_benchmark_results.csv
             *.png
             benchmark_report.md

jvector-examples/src/main/java/io/github/jbellis/jvector/example/AutoBenchYAML.java

@@ -98,14 +98,25 @@ public static void main(String[] args) throws IOException {
 
         logger.info("Heap space available is {}", Runtime.getRuntime().maxMemory());
 
-        // Filter out --output and its argument from the args
+        // Filter out --output, --config and their arguments from the args
         String finalOutputPath = outputPath;
+        String configPath = null;
+        for (int i = 0; i < args.length - 1; i++) {
+            if (args[i].equals("--config")) configPath = args[i+1];
+        }
+        String finalConfigPath = configPath;
         String[] filteredArgs = Arrays.stream(args)
-                .filter(arg -> !arg.equals("--output") && !arg.equals(finalOutputPath))
+                .filter(arg -> !arg.equals("--output") && !arg.equals(finalOutputPath) && 
+                               !arg.equals("--config") && !arg.equals(finalConfigPath))
                 .toArray(String[]::new);
 
+        // Log the filtered arguments for debugging
+        logger.info("Filtered arguments: {}", Arrays.toString(filteredArgs));
+
         // generate a regex that matches any regex in filteredArgs, or if filteredArgs is empty/null, match everything
         var regex = filteredArgs.length == 0 ? ".*" : Arrays.stream(filteredArgs).flatMap(s -> Arrays.stream(s.split("\\s"))).map(s -> "(?:" + s + ")").collect(Collectors.joining("|"));
+        logger.info("Generated regex pattern: {}", regex);
+
         // compile regex and do substring matching using find
         var pattern = Pattern.compile(regex);
 
@@ -125,7 +136,7 @@ public static void main(String[] args) throws IOException {
                     if (datasetName.endsWith(".hdf5")) {
                         datasetName = datasetName.substring(0, datasetName.length() - ".hdf5".length());
                     }
-                    
+
                     MultiConfig config = MultiConfig.getDefaultConfig("autoDefault");
                     config.dataset = datasetName;
                     logger.info("Using configuration: {}", config);
@@ -140,7 +151,7 @@ public static void main(String[] args) throws IOException {
                             config.search.getCompressorParameters(), 
                             config.search.topKOverquery, 
                             config.search.useSearchPruning));
-                    
+
                     logger.info("Benchmark completed for dataset: {}", datasetName);
                 } catch (Exception e) {
                     logger.error("Exception while processing dataset {}", datasetName, e);
@@ -159,20 +170,20 @@ public static void main(String[] args) throws IOException {
             mapper.writerWithDefaultPrettyPrinter().writeValue(detailsFile, results);
 
             File outputFile = new File(outputPath + ".csv");
-            
+
             // Get summary statistics by dataset
             Map<String, SummaryStats> statsByDataset = BenchmarkSummarizer.summarizeByDataset(results);
-            
+
             // Write CSV data
             try (FileWriter writer = new FileWriter(outputFile)) {
                 // Write CSV header
                 writer.write("dataset,QPS,Mean Latency,Recall@10\n");
-                
+
                 // Write one row per dataset with average metrics
                 for (Map.Entry<String, SummaryStats> entry : statsByDataset.entrySet()) {
                     String dataset = entry.getKey();
                     SummaryStats datasetStats = entry.getValue();
-                    
+
                     writer.write(dataset + ",");
                     writer.write(datasetStats.getAvgQps() + ",");
                     writer.write(datasetStats.getAvgLatency() + ",");