Add cmake "run" target and use it in PR CI

source/CMakeLists.txt

@@ -87,6 +87,11 @@
 #
 #       ./build/randbytes 10
 #
+# cmake also generates a run target.  The run target serves as a
+# unit test which runs all commands. Use 'cmake --build build -t run' command
+# to run it.  Note that if the build was configured without explicit
+# OPENSSL_ROOT_DIR set, one might wants to properly set run_certdir cache
+# variable beforehand.
 #
 # 32-bit builds
 # =============
@@ -128,6 +133,13 @@ add_library(perf perflib/perfhelper.c perflib/perfsslhelper.c perflib/threads.c
 
 if(WIN32)
     target_sources(perf PRIVATE perflib/getopt.c perflib/basename.c perflib/err.c)
+    #
+    # windows release build on github workflow places places .exe files under
+    # build\Release. This tweak hopes to place or .exe files under \build
+    # for both build types.
+    #
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG "${OUTPUT_DIRECTORY}")
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "${OUTPUT_DIRECTORY}")
 endif()
 
 target_include_directories(perf PUBLIC "${PROJECT_SOURCE_DIR}")
@@ -187,3 +199,254 @@ target_link_libraries(writeread PRIVATE perf)
 
 add_executable(evp_hash evp_hash.c)
 target_link_libraries(evp_hash PRIVATE perf)
+
+## Running tests
+# Options
+set(run_tests evp_fetch
+              evp_hash
+              evp_setpeer
+              handshake
+              newrawkey
+              pkeyread
+              providerdoall
+              randbytes
+              rsasign
+              rwlocks
+              sslnew
+              #ssl_poll_perf
+              writeread
+              x509storeissuer
+              CACHE STRING "List of tests to run")
+
+set(run_add_version_dep
+    ON
+    CACHE BOOL "Whether to print version information on run " +
+               "(call all the tests with -V option before the main run)")
+
+# Per-test options, the format: test option values
+set(run_evp_fetch_pqs
+    evp_fetch "" "" "-q"
+    CACHE STRING "Post-quantum option for evp_fetch")
+set(run_evp_hash_operations
+    evp_hash "" "" "-o deprecated" "-o evp_isolated" "-o evp_shared"
+    CACHE STRING "Modes of operation for evp_hash")
+set(run_evp_hash_update_times
+    evp_hash "" "" "-u 1" "-u 5"
+    CACHE STRING "Digest update times for evp_hash")
+set(run_evp_hash_algorithms
+    evp_hash "" "" "-a SHA1" "-a SHA224" "-a SHA256" "-a SHA384" "-a SHA512"
+    CACHE STRING "Digest hash algorithms for evp_hash")
+set(run_evp_setpeer_keys
+    evp_setpeer "-k" dh ec256 ec521 x25519 all
+    CACHE STRING "Key types for evp_setpeer")
+set(run_newrawkey_algos
+    newrawkey "-a" x25519 ml-kem-512 ml-kem-768 ml-kem-1024
+    CACHE STRING "Algorithms for newrawkey")
+set(run_pkeyread_keys
+    pkeyread "-k" dh dhx dsa ec rsa x25519 all
+    CACHE STRING "Key types for pkeyread")
+set(run_pkeyread_fmts
+    pkeyread "-f" pem der all
+    CACHE STRING "Key formats for pkeyread")
+set(run_handshake_pools
+    handshake "" "-p" "-P" "-l"
+    CACHE STRING "Pool types for handshake")
+set(run_handshake_ctx_sharing
+    handshake "" "" "-s"
+    CACHE STRING "Context sharing option for handshake")
+set(run_handshake_pool_size
+    handshake "" "" "-o 4" "-o 256"
+    CACHE STRING "Pool size for handshake")
+set(run_handshake_secure_memory
+    handshake "" "-S 1048576"
+    CACHE STRING "Secure memory usage for handshake")
+set(run_writeread_ctx_sharing
+    writeread "" "" "-s"
+    CACHE STRING "Context sharing for writeread")
+set(run_writeread_dtls
+    writeread "" "" "-d"
+    CACHE STRING "DTLS mode for writeread")
+set(run_writeread_buffers
+    writeread "" "" "-b 256" "-b 4096"
+    CACHE STRING "Buffer size for writeread")
+
+# The list of per-tet options
+set(run_opts run_evp_fetch_pqs
+             run_evp_hash_operations
+             run_evp_hash_update_times
+             run_evp_hash_algorithms
+             run_evp_setpeer_keys
+             run_newrawkey_algos
+             run_pkeyread_keys
+             run_pkeyread_fmts
+             run_handshake_pools
+             run_handshake_ctx_sharing
+             run_handshake_pool_size
+             run_handshake_secure_memory
+             run_writeread_ctx_sharing
+             run_writeread_dtls
+             run_writeread_buffers
+             CACHE STRING "List of per-text options")
+
+# Used across multiple tests
+set(run_certdir_tests handshake writeread x509storeissuer
+    CACHE STRING "List of tests that require certdir parameter")
+file(TO_NATIVE_PATH "${OPENSSL_ROOT_DIR}/test/certs/" run_certdir_def_path)
+set(run_certdir "${run_certdir_def_path}"
+    CACHE PATH "Path to certificates directory for tests that need it")
+
+# Common options
+set(run_terse "" "-t"
+    CACHE STRING "List of terse output options")
+set(run_threads 1 4
+    CACHE STRING "List of thread counts")
+
+add_custom_target(run
+                  COMMENT "Run perf tests"
+                  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+
+# Version target
+add_custom_target(run-version
+                  COMMENT "Get version information for perf tests"
+                  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+if (run_add_version_dep)
+    add_dependencies(run run-version)
+endif()
+
+
+if(WIN32)
+    # Prepare the PATH value for the generated batch files on Windows
+    get_target_property(crypto_path OpenSSL::Crypto IMPORTED_LOCATION)
+    get_filename_component(crypto_path "${crypto_path}" DIRECTORY)
+    get_target_property(ssl_path OpenSSL::SSL IMPORTED_LOCATION)
+    get_filename_component(ssl_path "${ssl_path}" DIRECTORY)
+    set(win32_path "${crypto_path};${ssl_path};%PATH%")
+endif()
+
+#
+# gen_run_target(<cmd> <test> [VAR <var>] [NAME <name>])
+#
+# Generates a target that runs <cmd> and depends on target <test> and sets
+# <var> to the target's name.
+#
+function(gen_run_taget _CMD _TEST)
+    include(CMakeParseArguments)
+
+    set(_Options "")
+    set(_OneValueArgs VAR NAME)
+    set(_MultiValueArgs "")
+    cmake_parse_arguments(GEN_RUN_TARGET
+        "${_Options}"
+        "${_OneValueArgs}"
+        "${_MultiValueArgs}"
+        ${ARGN}
+    )
+
+    #
+    # In order to execute the command on Windows, we need
+    # to add OPENSSL_ROOT_DIR to PATH variable so runtime
+    # linker can find OpenSSL 's .dll files.  Adjusting PATH on Windows
+    # requires us to generate a .bat file with two commands:
+    #    set PATH=c:\...\OPENSSL_ROOT_DIR;%PATH%
+    #    command_to_exec with_options
+    #
+    string(REGEX REPLACE "  *" ";" cmd "${_CMD}")
+    string(REGEX REPLACE "[^0-9A-Za-z]" "-" cmd_target_name "${cmd}")
+    # A hack for lesser OSes that cannot normally distinguish lower-
+    # and uppercase letters.
+    if (WIN32 OR APPLE)
+        string(REGEX REPLACE "[A-Z]" "\\0_" cmd_target_name
+               "${cmd_target_name}")
+    endif()
+    if (NOT GEN_RUN_TARGET_NAME)
+        set(GEN_RUN_TARGET_NAME "run-${cmd_target_name}")
+    endif()
+    string(REPLACE ";" " " cmd_desc "${cmd}")
+
+
+    if(WIN32)
+        #
+        # generate .bat file and target to execute it.
+        #
+        file(GENERATE OUTPUT "${cmd_target_name}.bat"
+             CONTENT "set PATH=${win32_path}\n${_CMD}"
+             NEWLINE_STYLE WIN32)
+        add_custom_target("${GEN_RUN_TARGET_NAME}"
+                          COMMAND "${cmd_target_name}.bat"
+                          DEPENDS "${_TEST}"
+                          COMMENT "Run ${cmd_desc}"
+                          WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+                          USES_TERMINAL)
+    else()
+        add_custom_target("${GEN_RUN_TARGET_NAME}"
+                          COMMAND ${cmd}
+                          DEPENDS "${_TEST}"
+                          COMMENT "Run ${cmd_desc}"
+                          WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+                          USES_TERMINAL)
+    endif()
+
+    if (GEN_RUN_TARGET_VAR)
+        set("${GEN_RUN_TARGET_VAR}" "${GEN_RUN_TARGET_NAME}" PARENT_SCOPE)
+    endif()
+endfunction()
+
+# run targets generation
+foreach(test IN LISTS run_tests)
+    set(cmds "${test}")
+
+    # test-specific options
+    foreach(opt_name IN LISTS run_opts)
+        set(opt "${${opt_name}}")
+        list(GET opt 0 test_name)
+        list(GET opt 1 test_opt)
+        list(REMOVE_AT opt 0 1)
+
+        if(test IN_LIST test_name)
+            set(new_cmds)
+            foreach(cmd IN LISTS cmds)
+                foreach(val IN LISTS opt)
+                    list(APPEND new_cmds "${cmd} ${test_opt} ${val}")
+                endforeach()
+            endforeach()
+            set(cmds ${new_cmds})
+        endif()
+    endforeach()
+
+    # terse
+    set(new_cmds)
+    foreach(cmd IN LISTS cmds)
+        foreach(val IN LISTS run_terse)
+            list(APPEND new_cmds "${cmd} ${val}")
+        endforeach()
+    endforeach()
+    set(cmds ${new_cmds})
+
+    # certdir
+    if(test IN_LIST run_certdir_tests)
+        set(new_cmds)
+        foreach(cmd IN LISTS cmds)
+            list(APPEND new_cmds "${cmd} ${run_certdir}")
+        endforeach()
+        set(cmds ${new_cmds})
+    endif()
+
+    # threads
+    set(new_cmds)
+    foreach(cmd IN LISTS cmds)
+        foreach(val IN LISTS run_threads)
+            list(APPEND new_cmds "${cmd} ${val}")
+        endforeach()
+    endforeach()
+    set(cmds ${new_cmds})
+
+    # Run target
+    foreach(cmd IN LISTS cmds)
+        gen_run_taget("${cmd}" "${test}" VAR run_target_name)
+        add_dependencies(run "${run_target_name}")
+    endforeach()
+
+    # Per-test version taget
+    gen_run_taget("${test} -V" "${test}" NAME "run-version-${test}")
+    add_dependencies(run-version "run-version-${test}")
+endforeach()

source/evp_fetch.c

@@ -181,12 +181,13 @@ void do_fetch(size_t num)
      * to be a multiple of the number of fetch entries therefore at the last
      * iteration we may not check all the algorithms.
      */
+    i = 0;
     do {
         /*
          * If we set a fetch type, always use that
          */
         if (exclusive_fetch_type == FETCH_END) {
-            j = i % array_size;
+            j = i++ % array_size;
             fetch_alg = fetch_entries[j].alg;
             j = fetch_entries[j].ftype;
         } else {
@@ -297,9 +298,9 @@ int main(int argc, char *argv[])
     int opt;
 
 #ifdef OPENSSL_DO_PQ
-    while ((opt = getopt(argc, argv, "tq")) != -1) {
+    while ((opt = getopt(argc, argv, "tqV")) != -1) {
 #else
-    while ((opt = getopt(argc, argv, "t")) != -1) {
+    while ((opt = getopt(argc, argv, "tV")) != -1) {
 #endif
         switch (opt) {
         case 't':
@@ -310,16 +311,20 @@ int main(int argc, char *argv[])
             pq = 1;
             break;
 #endif
+        case 'V':
+            perflib_print_version(basename(argv[0]));
+            return EXIT_SUCCESS;
         default:
 #ifdef OPENSSL_DO_PQ
-            printf("Usage: %s [-t] [-q] threadcount\n", basename(argv[0]));
+            printf("Usage: %s [-t] [-q] [-V] threadcount\n", basename(argv[0]));
 #else
-            printf("Usage: %s [-t] threadcount\n", basename(argv[0]));
+            printf("Usage: %s [-t] [-V] threadcount\n", basename(argv[0]));
 #endif
             printf("-t - terse output\n");
 #ifdef OPENSSL_DO_PQ
             printf("-q - include post-quantum algorithms\n");
 #endif
+            printf("-V - print version information and exit\n");
             return EXIT_FAILURE;
         }
     }

source/evp_hash.c

@@ -18,8 +18,10 @@
 #include <stdlib.h>
 #include <stdio.h>
 #ifndef _WIN32
+# include <libgen.h>
 # include <unistd.h>
 #else
+# include "perflib/basename.h"
 # include "perflib/getopt.h"
 #endif	/* _WIN32 */
 
@@ -246,12 +248,13 @@ static void do_hash_evp_shared(size_t num)
 
 static void print_help()
 {
-    printf("Usage: evp_hash [-h] [-t] [-o operation] [-u update-times] [-a algorithm] thread-count\n");
+    printf("Usage: evp_hash [-h] [-t] [-o operation] [-u update-times] [-a algorithm] [-V] thread-count\n");
     printf("-h - print this help output\n");
     printf("-t - terse output\n");
     printf("-o operation - mode of operation. One of [deprecated, evp_isolated, evp_shared] (default: evp_shared)\n");
     printf("-u update-times - times to update digest. 1 for one-shot (default: 1)\n");
     printf("-a algorithm - One of: [SHA1, SHA224, SHA256, SHA384, SHA512] (default: SHA1)\n");
+    printf("-V - print version information and exit\n");
     printf("thread-count - number of threads\n");
 }
 
@@ -263,7 +266,7 @@ int main(int argc, char *argv[])
     int terse = 0, operation = EVP_SHARED, hash_algorithm = SHA1_ALG;
     int j, opt, rc = EXIT_FAILURE;
 
-    while ((opt = getopt(argc, argv, "hto:u:a:")) != -1) {
+    while ((opt = getopt(argc, argv, "hto:u:a:V")) != -1) {
         switch (opt) {
         case 't':
             terse = 1;
@@ -305,7 +308,13 @@ int main(int argc, char *argv[])
                 goto out;
             }
             break;
+        case 'V':
+            perflib_print_version(basename(argv[0]));
+            rc = EXIT_SUCCESS;
+            goto out;
         case 'h':
+            rc = EXIT_SUCCESS;
+            /* Fallthrough */
         default:
             print_help();
             goto out;