KnnPerfTest Median Results Display (#425)

* initial commit

* Added runs parameter to command line, refactored code

* Removed median chart from single-test runs, added full summary output

* Reverted parameters

* Reformatted and reverted parameters

Author: aylonsk

gradle/knn.gradle

@@ -47,7 +47,10 @@ task runKnnPerfTest (type: Exec) {
 
     workingDir rootProject.getRootDir()
 
-    commandLine 'python3', 'src/python/knnPerfTest.py'
+    doFirst {
+        def runs = project.hasProperty('runs') ? project.property('runs') : '1'
+        commandLine 'python3', 'src/python/knnPerfTest.py', '--runs', runs
+    }
 }
 
 task extractVectorTasks (type: Copy) {

src/python/knnPerfTest.py

@@ -7,8 +7,10 @@
 #   - why only one thread
 #   - report net concurrency utilized in the table
 
+import argparse
 import multiprocessing
 import re
+import statistics
 import subprocess
 import sys
 
@@ -20,17 +22,20 @@
 
 # SETUP:
 ### Download and extract data files: Wikipedia line docs + GloVe
-# python src/python/setup.py -download
+# python src/python/initial_setup.py -download    OR    curl -O  https://downloads.cs.stanford.edu/nlp/data/glove.6B.zip -k
 # cd ../data
 # unzip glove.6B.zip
-# unlzma enwiki-20120502-lines-1k.txt.lzma
+# unlzma enwiki-20120502-lines-1k.txt.lzma    OR    xz enwiki-20120502-lines-1k.txt.lzma
 ### Create document and task vectors
 # ./gradlew vectors-100
 #
 # change the parameters below and then run (you can still manually run this file, but using gradle command
 # below will auto recompile if you made any changes to java files in luceneutils)
 # ./gradlew runKnnPerfTest
 #
+# for the median result of n runs with the same parameters:
+# ./gradlew runKnnPerfTest -Pruns=n
+#
 # you may want to modify the following settings:
 
 DO_PROFILING = False
@@ -135,9 +140,9 @@ def run_knn_benchmark(checkout, values):
   indexes = [0] * len(values.keys())
   indexes[-1] = -1
   args = []
-  # dim = 100
-  # doc_vectors = constants.GLOVE_VECTOR_DOCS_FILE
-  # query_vectors = '%s/luceneutil/tasks/vector-task-100d.vec' % constants.BASE_DIR
+  dim = 100
+  doc_vectors = "%s/lucene_util/tasks/enwiki-20120502-lines-1k-100d.vec" % constants.BASE_DIR
+  query_vectors = "%s/lucene_util/tasks/vector-task-100d.vec" % constants.BASE_DIR
   # dim = 768
   # doc_vectors = '/lucenedata/enwiki/enwiki-20120502-lines-1k-mpnet.vec'
   # query_vectors = '/lucenedata/enwiki/enwiki-20120502.mpnet.vec'
@@ -153,9 +158,9 @@ def run_knn_benchmark(checkout, values):
   # query_vectors = '/d/electronics_query_vectors.bin'
 
   # Cohere dataset
-  dim = 768
-  doc_vectors = f"{constants.BASE_DIR}/data/cohere-wikipedia-docs-{dim}d.vec"
-  query_vectors = f"{constants.BASE_DIR}/data/cohere-wikipedia-queries-{dim}d.vec"
+  # dim = 768
+  # doc_vectors = f"{constants.BASE_DIR}/data/cohere-wikipedia-docs-{dim}d.vec"
+  # query_vectors = f"{constants.BASE_DIR}/data/cohere-wikipedia-queries-{dim}d.vec"
   # doc_vectors = f"/lucenedata/enwiki/{'cohere-wikipedia'}-docs-{dim}d.vec"
   # query_vectors = f"/lucenedata/enwiki/{'cohere-wikipedia'}-queries-{dim}d.vec"
   parentJoin_meta_file = f"{constants.BASE_DIR}/data/{'cohere-wikipedia'}-metadata.csv"
@@ -285,6 +290,7 @@ def run_knn_benchmark(checkout, values):
 
   print_fixed_width(all_results, skip_headers)
   print_chart(all_results)
+  return all_results, skip_headers
 
 
 def print_fixed_width(all_results, columns_to_skip):
@@ -425,7 +431,55 @@ def chart_args_label(args):
   return str(args)
 
 
+def run_n_knn_benchmarks(LUCENE_CHECKOUT, PARAMS, n):
+  rec, lat, net, avg = [], [], [], []
+  tests = []
+  for i in range(n):
+    results, skip_headers = run_knn_benchmark(LUCENE_CHECKOUT, PARAMS)
+    tests.append(results)
+    first_4_numbers = results[0][0].split("\t")[:4]
+    first_4_numbers = [float(num) for num in first_4_numbers]
+
+    # store relevant data points
+    rec.append(first_4_numbers[0])
+    lat.append(first_4_numbers[1])
+    net.append(first_4_numbers[2])
+    avg.append(first_4_numbers[3])
+
+  # reconstruct string with median results
+  med_results = []
+  med_string = ""
+  med_string += f"{round(statistics.median(rec), 3)}\t"
+  med_string += f"{round(statistics.median(lat), 3)}\t"
+  med_string += f"{round(statistics.median(net), 3)}\t"
+  med_string += f"{round(statistics.median(avg), 3)}\t"
+
+  split_results = results[0][0].split("\t")
+  split_string = "\t".join(split_results[4:])
+  med_string += split_string
+  med_tuple = (med_string, results[0][1])
+  med_results.append(med_tuple)
+
+  # re-print all tables in a row
+  print("\nFinal Results:")
+  for i in range(n):
+    print(f"\nTest {i + 1}:")
+    print_fixed_width(tests[i], skip_headers)
+
+  # print median results in table
+  print("\nMedian Results:")
+  print_chart(med_results)
+  print_fixed_width(med_results, skip_headers)
+
+
 if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Run KNN benchmarks")
+  parser.add_argument("--runs", type=int, default=1, help="Number of times to run the benchmark (default: 1)")
+  n = parser.parse_args()
+
   # Where the version of Lucene is that will be tested. Now this will be sourced from gradle.properties
   LUCENE_CHECKOUT = getLuceneDirFromGradleProperties()
-  run_knn_benchmark(LUCENE_CHECKOUT, PARAMS)
+  if n.runs == 1:
+    run_knn_benchmark(LUCENE_CHECKOUT, PARAMS)
+  else:
+    run_n_knn_benchmarks(LUCENE_CHECKOUT, PARAMS, n.runs)