Add Benchmarks (#11)

* Add Bash script for running benchmarks

* Update `test.sh`

* Update with more scripts

* Update with new parallel read benchmark

* Update to add missing header

* Update

Author: BenBrock

examples/CMakeLists.txt

@@ -12,4 +12,5 @@ add_example(bsp2mtx)
 add_example(check_equivalence)
 add_example(bsp-ls)
 add_example(benchmark_read)
+add_example(benchmark_read_parallel)
 add_example(benchmark_write)

examples/benchmark_read.c

@@ -42,12 +42,14 @@ double compute_variance(double* x, size_t n) {
 
 void flush_cache() {
 #ifdef __APPLE__
-  system("bash -c \"sync && sudo purge\"");
+  int rv = system("bash -c \"sync && sudo purge\"");
 #elif __linux__
-  system("bash -c \"sync\" && sudo echo 3 > /proc/sys/vm/drop_caches");
+  int rv = system("bash -c \"sync\" && sudo sh -c \"/usr/bin/echo 3 > "
+                  "/proc/sys/vm/drop_caches\"");
 #else
   static_assert(false);
 #endif
+  usleep(100000);
 }
 
 int main(int argc, char** argv) {
@@ -60,14 +62,20 @@ int main(int argc, char** argv) {
 
   printf("Opening %s\n", file_name);
 
-  const int num_trials = 1;
+  const int num_trials = 10;
 
   double durations[num_trials];
 
   size_t nbytes = 0;
 
   // To flush the filesystem cache before each trial, change to `true`.
-  bool cold_cache = false;
+  bool cold_cache = true;
+
+  // If running warm cache experiments, read once to warm cache.
+  if (!cold_cache) {
+    bsp_matrix_t mat = bsp_read_matrix(file_name, NULL);
+    bsp_destroy_matrix_t(mat);
+  }
 
   for (size_t i = 0; i < num_trials; i++) {
     if (cold_cache) {
@@ -79,6 +87,10 @@ int main(int argc, char** argv) {
     durations[i] = end - begin;
     nbytes = bsp_matrix_nbytes(mat);
     bsp_destroy_matrix_t(mat);
+
+    double gbytes = ((double) nbytes) / 1024 / 1024 / 1024;
+    double gbytes_s = gbytes / durations[i];
+    printf("FORPARSER: %s,%lf,%lf\n", file_name, durations[i], gbytes_s);
   }
 
   printf("[");
@@ -117,7 +129,5 @@ int main(int argc, char** argv) {
   }
   printf("]\n");
 
-  printf("FORPARSER: %s,%lf,%lf\n", file_name, median_time, gbytes_s);
-
   return 0;
 }

examples/benchmark_read_parallel.c

@@ -0,0 +1,137 @@
+#include <binsparse/binsparse.h>
+#include <stdlib.h>
+#include <time.h>
+
+double gettime() {
+  struct timespec time;
+  clock_gettime(CLOCK_MONOTONIC, &time);
+  return ((double) time.tv_sec) + ((double) 1e-9) * time.tv_nsec;
+}
+
+int compar(const void* a, const void* b) {
+  double x = *((const double*) a);
+  double y = *((const double*) b);
+
+  double diff = x - y;
+
+  if (diff > 0) {
+    return 1;
+  } else if (diff < 0) {
+    return -1;
+  } else {
+    return 0;
+  }
+}
+
+double compute_variance(double* x, size_t n) {
+  double sum = 0;
+
+  for (size_t i = 0; i < n; i++) {
+    sum += x[i];
+  }
+
+  double mean = sum / n;
+
+  double sum_of_squares = 0;
+  for (size_t i = 0; i < n; i++) {
+    sum_of_squares += (x[i] - mean) * (x[i] - mean);
+  }
+
+  return sum_of_squares / (n - 1);
+}
+
+void flush_cache() {
+#ifdef __APPLE__
+  int rv = system("bash -c \"sync && sudo purge\"");
+#elif __linux__
+  int rv = system("bash -c \"sync\" && sudo sh -c \"/usr/bin/echo 3 > "
+                  "/proc/sys/vm/drop_caches\"");
+#else
+  static_assert(false);
+#endif
+  usleep(100000);
+}
+
+int main(int argc, char** argv) {
+  if (argc < 2) {
+    fprintf(stderr, "usage: ./benchmark_read [file_name.h5]\n");
+    return 1;
+  }
+
+  char* file_name = argv[1];
+
+  printf("Opening %s\n", file_name);
+
+  const int num_trials = 2;
+
+  int num_threads = 6;
+
+  double durations[num_trials];
+
+  size_t nbytes = 0;
+
+  // To flush the filesystem cache before each trial, change to `true`.
+  bool cold_cache = true;
+
+  // If running warm cache experiments, read once to warm cache.
+  if (!cold_cache && false) {
+    printf("Warm cache read...\n");
+    bsp_matrix_t mat = bsp_read_matrix_parallel(file_name, NULL, num_threads);
+    bsp_destroy_matrix_t(mat);
+  }
+
+  for (size_t i = 0; i < num_trials; i++) {
+    if (cold_cache) {
+      flush_cache();
+    }
+    double begin = gettime();
+    bsp_matrix_t mat = bsp_read_matrix_parallel(file_name, NULL, num_threads);
+    double end = gettime();
+    durations[i] = end - begin;
+    nbytes = bsp_matrix_nbytes(mat);
+
+    bsp_destroy_matrix_t(mat);
+
+    double gbytes = ((double) nbytes) / 1024 / 1024 / 1024;
+    double gbytes_s = gbytes / durations[i];
+    printf("FORPARSER: %s,%lf,%lf\n", file_name, durations[i], gbytes_s);
+  }
+
+  printf("[");
+  for (size_t i = 0; i < num_trials; i++) {
+    printf("%lf", durations[i]);
+    if (i + 1 < num_trials) {
+      printf(", ");
+    }
+  }
+  printf("]\n");
+
+  qsort(durations, num_trials, sizeof(double), compar);
+
+  double variance = compute_variance(durations, num_trials);
+
+  double median_time = durations[num_trials / 2];
+
+  printf("Read file in %lf seconds\n", median_time);
+
+  if (num_trials > 1) {
+    printf("Variance is %lf seconds, standard devication is %lf seconds\n",
+           variance, sqrt(variance));
+  }
+
+  double gbytes = ((double) nbytes) / 1024 / 1024 / 1024;
+  double gbytes_s = gbytes / median_time;
+
+  printf("Achieved %lf GiB/s\n", gbytes_s);
+
+  printf("[");
+  for (size_t i = 0; i < num_trials; i++) {
+    printf("%lf", durations[i]);
+    if (i + 1 < num_trials) {
+      printf(", ");
+    }
+  }
+  printf("]\n");
+
+  return 0;
+}

examples/benchmark_write.c

@@ -42,19 +42,21 @@ double compute_variance(double* x, size_t n) {
 
 void flush_cache() {
 #ifdef __APPLE__
-  system("bash -c \"sync && sudo purge\"");
+  int rv = system("bash -c \"sync && sudo purge\"");
 #elif __linux__
-  system("bash -c \"sync\" && sudo echo 3 > /proc/sys/vm/drop_caches");
+  int rv = system("bash -c \"sync\" && sudo sh -c \"/usr/bin/echo 3 > "
+                  "/proc/sys/vm/drop_caches\"");
 #else
   static_assert(false);
 #endif
+  usleep(100000);
 }
 
 void flush_writes() {
 #ifdef __APPLE__
-  system("bash -c \"sync\"");
+  int rv = system("bash -c \"sync\"");
 #elif __linux__
-  system("bash -c \"sync\"");
+  int rv = system("bash -c \"sync\"");
 #else
   static_assert(false);
 #endif
@@ -63,7 +65,7 @@ void flush_writes() {
 void delete_file(const char* file_name) {
   char command[2048];
   snprintf(command, 2047, "rm %s", file_name);
-  system(command);
+  int rv = system(command);
 }
 
 int main(int argc, char** argv) {
@@ -85,7 +87,7 @@ int main(int argc, char** argv) {
 
   printf("Opening %s\n", file_name);
 
-  const int num_trials = 1;
+  const int num_trials = 10;
 
   double durations[num_trials];
 
@@ -105,7 +107,7 @@ int main(int argc, char** argv) {
 
   // To flush each write to the filesystem and include this in the timing,
   // change to `true`.
-  bool flush_each_write = true;
+  bool flush_each_write = false;
 
   for (size_t i = 0; i < num_trials; i++) {
     if (cold_cache) {
@@ -125,6 +127,11 @@ int main(int argc, char** argv) {
     double end = gettime();
     durations[i] = end - begin;
 
+    double gbytes = ((double) nbytes) / 1024 / 1024 / 1024;
+    double gbytes_s = gbytes / durations[i];
+
+    printf("FORPARSER: %s,%lf,%lf\n", file_name, durations[i], gbytes_s);
+
     delete_file(output_filename);
   }
 
@@ -164,7 +171,5 @@ int main(int argc, char** argv) {
   }
   printf("]\n");
 
-  printf("FORPARSER: %s,%lf,%lf\n", file_name, median_time, gbytes_s);
-
   return 0;
 }

include/binsparse/array.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <assert.h>
+#include <binsparse/detail/shm_tools.h>
 #include <binsparse/types.h>
 #include <stdlib.h>
 #include <string.h>
@@ -9,12 +10,15 @@ typedef struct bsp_array_t {
   void* data;
   size_t size;
   bsp_type_t type;
+  bool shmat_memory;
+  bsp_shm_t shm;
 } bsp_array_t;
 
 bsp_array_t bsp_construct_default_array_t() {
   bsp_array_t array;
   array.data = NULL;
   array.size = 0;
+  array.shmat_memory = false;
   return array;
 }
 
@@ -23,8 +27,10 @@ bsp_array_t bsp_construct_array_t(size_t size, bsp_type_t type) {
 
   bsp_array_t array;
   array.data = malloc(byte_size);
+  assert(array.data != NULL);
   array.size = size;
   array.type = type;
+  array.shmat_memory = false;
 
   return array;
 }
@@ -70,7 +76,37 @@ bsp_array_t bsp_fp_array_to_complex(bsp_array_t other) {
 }
 
 void bsp_destroy_array_t(bsp_array_t array) {
-  free(array.data);
+  if (array.shmat_memory == false) {
+    free(array.data);
+  } else {
+    bsp_shm_detach(array.data);
+  }
+}
+
+bool bsp_array_equal(bsp_array_t x, bsp_array_t y) {
+  if (x.size != y.size) {
+    return false;
+  }
+
+  if (x.size == 0) {
+    return true;
+  }
+
+  if (x.type != y.type) {
+    return false;
+  }
+
+  uint8_t* x_data = (uint8_t*) x.data;
+  uint8_t* y_data = (uint8_t*) y.data;
+  for (size_t i = 0; i < x.size * bsp_type_size(x.type); i++) {
+    if (x_data[i] != y_data[i]) {
+      printf("Index %zu incorrect %d != %d\n", i, (int) x_data[i],
+             (int) y_data[i]);
+      fflush(stdout);
+      return false;
+    }
+  }
+  return true;
 }
 
 #ifndef __cplusplus

include/binsparse/detail/shm_tools.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <stddef.h>
+#include <stdio.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+typedef struct {
+  int id;
+  size_t size;
+} bsp_shm_t;
+
+bsp_shm_t bsp_shm_new(size_t size) {
+  bsp_shm_t shm;
+  shm.size = size;
+
+  if ((shm.id = shmget(IPC_PRIVATE, size,
+                       IPC_CREAT | IPC_EXCL | S_IRUSR | S_IWUSR)) < 0) {
+    perror("shmget");
+  }
+
+  return shm;
+}
+
+void bsp_shm_delete(bsp_shm_t shm) {
+  shmctl(shm.id, IPC_RMID, 0);
+}
+
+void* bsp_shm_attach(bsp_shm_t shm) {
+  void* data;
+
+  if ((data = shmat(shm.id, NULL, 0)) == (void*) -1) {
+    perror("write");
+  }
+
+  return data;
+}
+
+void bsp_shm_detach(void* data) {
+  shmdt(data);
+}