Set a constant amount of runtime on more perf tests

Update some more perf tests to run for a set amount of time, similar to #30. This PR resolves the first half of the issues in the epic openssl/project#1251

Fixes openssl/project#1252
Fixes openssl/project#1253
Fixes openssl/project#1254
Fixes openssl/project#1255
Fixes openssl/project#1256
Fixes openssl/project#1257

Reviewed-by: Tomas Mraz <[email protected]>
Reviewed-by: Saša Nedvědický <[email protected]>
Reviewed-by: Neil Horman <[email protected]>
(Merged from #33)

Author: andrewkdinh

source/evp_fetch.c

@@ -31,7 +31,7 @@
 # define OPENSSL_DO_PQ
 #endif
 
-#define NUM_CALLS_PER_TEST         10000000
+#define RUN_TIME 5
 
 /*
  * Update the constant numbers below if you add or remove
@@ -58,7 +58,8 @@
 #define FETCH_ENTRY_PQ_ALGS_N      \
     (FETCH_ENTRY_ML_KEM_N + FETCH_ENTRY_ML_DSA_N + FETCH_ENTRY_SLH_DSA_N)
 
-OSSL_TIME *times;
+size_t *counts;
+OSSL_TIME max_time;
 
 int err = 0;
 int pq = 0;
@@ -158,7 +159,7 @@ static struct fetch_data_entry fetch_entries[] = {
 
 void do_fetch(size_t num)
 {
-    OSSL_TIME start, end;
+    OSSL_TIME time;
     size_t i, j;
     const char *fetch_alg = NULL;
     int array_size = ARRAY_SIZE(fetch_entries);
@@ -171,7 +172,7 @@ void do_fetch(size_t num)
         array_size -= FETCH_ENTRY_PQ_ALGS_N;
     }
 
-    start = ossl_time_now();
+    counts[num] = 0;
 
     /*
      * Going through the fetch entries num_calls / threadcount times.
@@ -180,7 +181,7 @@ void do_fetch(size_t num)
      * to be a multiple of the number of fetch entries therefore at the last
      * iteration we may not check all the algorithms.
      */
-    for (i = 0; i < num_calls / threadcount; i++) {
+    do {
         /*
          * If we set a fetch type, always use that
          */
@@ -278,15 +279,16 @@ void do_fetch(size_t num)
             err = 1;
             return;
         }
+        counts[num]++;
+        time = ossl_time_now();
     }
-    end = ossl_time_now();
-    times[num] = ossl_time_subtract(end, start);
+    while (time.t < max_time.t);
 }
 
 int main(int argc, char *argv[])
 {
     OSSL_TIME duration;
-    OSSL_TIME ttime;
+    size_t total_count = 0;
     double av;
     int terse = 0;
     size_t i;
@@ -352,17 +354,16 @@ int main(int argc, char *argv[])
         printf("threadcount must be > 0\n");
         return EXIT_FAILURE;
     }
-    num_calls = NUM_CALLS_PER_TEST;
-    if (NUM_CALLS_PER_TEST % threadcount > 0) /* round up */
-        num_calls += threadcount - NUM_CALLS_PER_TEST % threadcount;
+
+    max_time = ossl_time_add(ossl_time_now(), ossl_seconds2time(RUN_TIME));
 
     ctx = OSSL_LIB_CTX_new();
     if (ctx == NULL)
         return EXIT_FAILURE;
 
-    times = OPENSSL_malloc(sizeof(OSSL_TIME) * threadcount);
-    if (times == NULL) {
-        printf("Failed to create times array\n");
+    counts = OPENSSL_malloc(sizeof(size_t) * threadcount);
+    if (counts == NULL) {
+        printf("Failed to create counts array\n");
         return EXIT_FAILURE;
     }
 
@@ -376,9 +377,8 @@ int main(int argc, char *argv[])
         goto out;
     }
 
-    ttime = times[0];
-    for (i = 1; i < threadcount; i++)
-        ttime = ossl_time_add(ttime, times[i]);
+    for (i = 0; i < threadcount; i++)
+        total_count += counts[i];
 
     /*
      * EVP_fetch_* calls are pretty fast, running in
@@ -388,7 +388,7 @@ int main(int argc, char *argv[])
      * zero in the math.  Instead, manually do the division, casting
      * our values as doubles so that we compute the proper time
      */
-    av = ((double)ossl_time2ticks(ttime) / num_calls) /(double)OSSL_TIME_US;
+    av = (double)RUN_TIME * 1e6 * threadcount / total_count;
 
     if (terse)
         printf("%lf\n", av);
@@ -398,6 +398,6 @@ int main(int argc, char *argv[])
     rc = EXIT_SUCCESS;
 out:
     OSSL_LIB_CTX_free(ctx);
-    OPENSSL_free(times);
+    OPENSSL_free(counts);
     return rc;
 }

source/evp_setpeer.c

@@ -22,20 +22,21 @@
 /* run 'make regen_key_samples' if header file is missing */
 #include "keys_setpeer.h"
 
-#define NUM_CALLS_PER_TEST         10000
+#define RUN_TIME 5
 
 int err = 0;
 
 size_t num_calls;
 static int threadcount;
-static OSSL_TIME *times = NULL;
+size_t *counts;
 
 EVP_PKEY *pkey = NULL;
+OSSL_TIME max_time;
 
 void do_setpeer(size_t num)
 {
     size_t i;
-    OSSL_TIME start, end;
+    OSSL_TIME time;
 
     EVP_PKEY_CTX *pkey_ctx = NULL;
 
@@ -53,17 +54,16 @@ void do_setpeer(size_t num)
         return;
     }
 
-    start = ossl_time_now();
+    counts[num] = 0;
 
-    for (i = 0; i < num_calls / threadcount; i++) {
+    do {
         if (EVP_PKEY_derive_set_peer(pkey_ctx, pkey) <= 0) {
             err = 1;
             break;
         }
-    }
-
-    end = ossl_time_now();
-    times[num] = ossl_time_subtract(end, start);
+        counts[num]++;
+        time = ossl_time_now();
+    } while (time.t < max_time.t);
 
     EVP_PKEY_CTX_free(pkey_ctx);
 }
@@ -84,15 +84,13 @@ static int sample_name_to_id(const char *sample_name)
 static double get_avcalltime(void)
 {
     int i;
-    OSSL_TIME t;
+    size_t total_count = 0;
     double avcalltime;
 
-    memset(&t, 0, sizeof(t));
     for (i = 0; i < threadcount; i++)
-        t = ossl_time_add(t, times[i]);
-    avcalltime = (double)ossl_time2ticks(t) / num_calls;
+        total_count += counts[i];
 
-    avcalltime =  avcalltime / (double)OSSL_TIME_US;
+    avcalltime = (double)RUN_TIME * 1e6 * threadcount / total_count;
 
     return avcalltime;
 }
@@ -132,7 +130,7 @@ static void usage(char * const argv[])
 int main(int argc, char *argv[])
 {
     OSSL_TIME duration;
-    OSSL_TIME ttime;
+    size_t total_count = 0;
     double avcalltime;
     int terse = 0;
     int rc = EXIT_FAILURE;
@@ -181,13 +179,11 @@ int main(int argc, char *argv[])
         return EXIT_FAILURE;
     }
 
-    num_calls = NUM_CALLS_PER_TEST;
-    if (NUM_CALLS_PER_TEST % threadcount > 0) /* round up */
-        num_calls += threadcount - NUM_CALLS_PER_TEST % threadcount;
+    max_time = ossl_time_add(ossl_time_now(), ossl_seconds2time(RUN_TIME));
 
-    times = OPENSSL_malloc(sizeof(OSSL_TIME) * threadcount);
-    if (times == NULL) {
-        printf("Failed to create times array\n");
+    counts = OPENSSL_malloc(sizeof(OSSL_TIME) * threadcount);
+    if (counts == NULL) {
+        printf("Failed to create counts array\n");
         return EXIT_FAILURE;
     }
 
@@ -241,6 +237,6 @@ int main(int argc, char *argv[])
 
     rc = EXIT_SUCCESS;
 out:
-    OPENSSL_free(times);
+    OPENSSL_free(counts);
     return rc;
 }

source/newrawkey.c

@@ -21,10 +21,11 @@
 #include <openssl/evp.h>
 #include "perflib/perflib.h"
 
-#define NUM_CALLS_PER_TEST         1000000
+#define RUN_TIME 5
 
 size_t num_calls;
-OSSL_TIME *times;
+size_t *counts;
+OSSL_TIME max_time;
 
 enum {
     ALGO_X25519,
@@ -336,7 +337,7 @@ void do_newrawkey(size_t num)
 {
     size_t i;
     EVP_PKEY *pkey;
-    OSSL_TIME start, end;
+    OSSL_TIME time;
     const unsigned char *key_data = key_x25519;
     size_t key_len = sizeof(key_x25519);
 
@@ -362,9 +363,9 @@ void do_newrawkey(size_t num)
             break;
     }
 
-    start = ossl_time_now();
+    counts[num] = 0;
 
-    for (i = 0; i < num_calls / threadcount; i++) {
+    do {
 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
         pkey = EVP_PKEY_new_raw_public_key_ex(NULL, alg_name, NULL, key_data,
                                               key_len);
@@ -379,16 +380,15 @@ void do_newrawkey(size_t num)
             err = 1;
         else
             EVP_PKEY_free(pkey);
-    }
-
-    end = ossl_time_now();
-    times[num] = ossl_time_subtract(end, start);
+        counts[num]++;
+        time = ossl_time_now();
+    } while (time.t < max_time.t);
 }
 
 int main(int argc, char *argv[])
 {
     OSSL_TIME duration;
-    OSSL_TIME ttime;
+    size_t total_count = 0;
     double av;
     int terse = 0;
     size_t i;
@@ -438,13 +438,11 @@ int main(int argc, char *argv[])
         printf("threadcount must be > 0\n");
         return EXIT_FAILURE;
     }
-    num_calls = NUM_CALLS_PER_TEST;
-    if (NUM_CALLS_PER_TEST % threadcount > 0) /* round up */
-        num_calls += threadcount - NUM_CALLS_PER_TEST % threadcount;
+    max_time = ossl_time_add(ossl_time_now(), ossl_seconds2time(RUN_TIME));
 
-    times = OPENSSL_malloc(sizeof(OSSL_TIME) * threadcount);
-    if (times == NULL) {
-        printf("Failed to create times array\n");
+    counts = OPENSSL_malloc(sizeof(size_t) * threadcount);
+    if (counts == NULL) {
+        printf("Failed to create counts array\n");
         return EXIT_FAILURE;
     }
 
@@ -458,9 +456,8 @@ int main(int argc, char *argv[])
         goto out;
     }
 
-    ttime = times[0];
-    for (i = 1; i < threadcount; i++)
-        ttime = ossl_time_add(ttime, times[i]);
+    for (i = 0; i < threadcount; i++)
+        total_count += counts[i];
 
     /*
      * EVP_PKEY_new_raw_public_key is pretty fast, running in
@@ -470,7 +467,7 @@ int main(int argc, char *argv[])
      * zero in the math.  Instead, manually do the division, casting
      * our values as doubles so that we compute the proper time
      */
-    av = ((double)ossl_time2ticks(ttime) / num_calls) /(double)OSSL_TIME_US;
+    av = (double)RUN_TIME * 1e6 * threadcount/ total_count;
 
     if (terse)
         printf("%lf\n", av);
@@ -480,6 +477,6 @@ int main(int argc, char *argv[])
 
     rc = EXIT_SUCCESS;
 out:
-    OPENSSL_free(times);
+    OPENSSL_free(counts);
     return rc;
 }