Add cmake "run" target and use it in PR CI

source/CMakeLists.txt

@@ -87,6 +87,11 @@
 #
 #       ./build/randbytes 10
 #
+# cmake also generates a run target.  The run target serves as a
+# unit test which runs all commands. Use 'cmake --build build -t run' command
+# to run it.  Note that if the build was configured without explicit
+# OPENSSL_ROOT_DIR set, one might wants to properly set run_certdir cache
+# variable beforehand.
 #
 # 32-bit builds
 # =============
@@ -128,6 +133,13 @@ add_library(perf perflib/perfhelper.c perflib/perfsslhelper.c perflib/threads.c
 
 if(WIN32)
     target_sources(perf PRIVATE perflib/getopt.c perflib/basename.c perflib/err.c)
+    #
+    # windows release build on github workflow places places .exe files under
+    # build\Release. This tweak hopes to place or .exe files under \build
+    # for both build types.
+    #
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG "${OUTPUT_DIRECTORY}")
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "${OUTPUT_DIRECTORY}")
 endif()
 
 target_include_directories(perf PUBLIC "${PROJECT_SOURCE_DIR}")
@@ -187,3 +199,258 @@ target_link_libraries(writeread PRIVATE perf)
 
 add_executable(evp_hash evp_hash.c)
 target_link_libraries(evp_hash PRIVATE perf)
+
+## Running tests
+# Options
+set(run_tests evp_fetch
+              evp_hash
+              evp_setpeer
+              handshake
+              newrawkey
+              pkeyread
+              providerdoall
+              randbytes
+              rsasign
+              rwlocks
+              sslnew
+              #ssl_poll_perf
+              writeread
+              x509storeissuer
+              CACHE STRING "List of tests to run")
+
+set(run_add_version_dep ON
+    CACHE BOOL "Whether to print version information on run " +
+               "(call all the tests with -V option before the main run)")
+
+# Per-test options, the format: test option values
+set(run_evp_fetch_pqs
+    evp_fetch "" "" "-q"
+    CACHE STRING "Post-quantum option for evp_fetch")
+set(run_evp_hash_operations
+    evp_hash "" "" "-o deprecated" "-o evp_isolated" "-o evp_shared"
+    CACHE STRING "Modes of operation for evp_hash")
+set(run_evp_hash_update_times
+    evp_hash "" "" "-u 1" "-u 5"
+    CACHE STRING "Digest update times for evp_hash")
+set(run_evp_hash_algorithms
+    evp_hash "" "" "-a SHA1" "-a SHA224" "-a SHA256" "-a SHA384" "-a SHA512"
+    CACHE STRING "Digest hash algorithms for evp_hash")
+set(run_evp_setpeer_keys
+    evp_setpeer "-k" dh ec256 ec521 x25519 all
+    CACHE STRING "Key types for evp_setpeer")
+set(run_newrawkey_algos
+    newrawkey "-a" x25519 ml-kem-512 ml-kem-768 ml-kem-1024
+    CACHE STRING "Algorithms for newrawkey")
+set(run_pkeyread_keys
+    pkeyread "-k" dh dhx dsa ec rsa x25519 all
+    CACHE STRING "Key types for pkeyread")
+set(run_pkeyread_fmts
+    pkeyread "-f" pem der all
+    CACHE STRING "Key formats for pkeyread")
+set(run_handshake_pools
+    handshake "" "-p" "-P" "-l"
+    CACHE STRING "Pool types for handshake")
+set(run_handshake_ctx_sharing
+    handshake "" "" "-s"
+    CACHE STRING "Context sharing option for handshake")
+set(run_handshake_pool_size
+    handshake "" "" "-o 4" "-o 256"
+    CACHE STRING "Pool size for handshake")
+set(run_handshake_secure_memory
+    handshake "" "-S 1048576"
+    CACHE STRING "Secure memory usage for handshake")
+set(run_writeread_ctx_sharing
+    writeread "" "" "-s"
+    CACHE STRING "Context sharing for writeread")
+set(run_writeread_dtls
+    writeread "" "" "-d"
+    CACHE STRING "DTLS mode for writeread")
+set(run_writeread_buffers
+    writeread "" "" "-b 256" "-b 4096"
+    CACHE STRING "Buffer size for writeread")
+
+# The list of per-tet options
+set(run_opts run_evp_fetch_pqs
+             run_evp_hash_operations
+             run_evp_hash_update_times
+             run_evp_hash_algorithms
+             run_evp_setpeer_keys
+             run_newrawkey_algos
+             run_pkeyread_keys
+             run_pkeyread_fmts
+             run_handshake_pools
+             run_handshake_ctx_sharing
+             run_handshake_pool_size
+             run_handshake_secure_memory
+             run_writeread_ctx_sharing
+             run_writeread_dtls
+             run_writeread_buffers
+             CACHE STRING "List of per-text options")
+
+# Used across multiple tests
+set(run_certdir_tests handshake writeread x509storeissuer
+    CACHE STRING "List of tests that require certdir parameter")
+file(TO_NATIVE_PATH "${OPENSSL_ROOT_DIR}/test/certs/" run_certdir_def_path)
+set(run_certdir "${run_certdir_def_path}"
+    CACHE PATH "Path to certificates directory for tests that need it")
+
+# Common options
+set(run_terse "" "-t"
+    CACHE STRING "List of terse output options")
+set(run_threads 1 4
+    CACHE STRING "List of thread counts")
+
+add_custom_target(run
+                  COMMENT "Run perf tests"
+                  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+
+# Version target
+add_custom_target(run-version
+                  COMMENT "Get version information for perf tests"
+                  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+if (run_add_version_dep)
+    add_dependencies(run run-version)
+endif()
+
+#
+# On Windows, we can't have nice things, and the easiest way to use
+# specifically the libraries we've built against is to copy them
+# in the directory where executable reside.
+#
+if (WIN32)
+    #
+    # An intermediate target to ensure that any DLLs touching happens strictly
+    # before their usage.
+    #
+    add_custom_target(run-copy-dlls
+                      COMMENT "Copy dependent DLLs into the build directory")
+
+    # Generate search path for later DLL copying
+    set(dll_search_dirs)
+    foreach (f IN LISTS OPENSSL_CRYPTO_LIBRARIES OPENSSL_SSL_LIBRARIES)
+        get_filename_component(f_dir "${f}" DIRECTORY)
+        list(APPEND dll_search_dirs "${f_dir}")
+    endforeach()
+endif()
+
+#
+# gen_run_target(<cmd> <test> [VAR <var>] [NAME <name>])
+#
+# Generates a target that runs <cmd> and depends on target <test> and sets
+# <var> to the target's name.
+#
+function(gen_run_taget _CMD _TEST)
+    include(CMakeParseArguments)
+
+    set(_Options "")
+    set(_OneValueArgs VAR NAME)
+    set(_MultiValueArgs "")
+    cmake_parse_arguments(GEN_RUN_TARGET
+        "${_Options}"
+        "${_OneValueArgs}"
+        "${_MultiValueArgs}"
+        ${ARGN}
+    )
+
+    string(REGEX REPLACE "  *" ";" cmd "${_CMD}")
+    string(REGEX REPLACE "[^0-9A-Za-z]" "-" cmd_target_name "${cmd}")
+    # A hack for lesser OSes that cannot normally distinguish lower-
+    # and uppercase letters.
+    if (WIN32 OR APPLE)
+        string(REGEX REPLACE "[A-Z]" "\\0_" cmd_target_name
+               "${cmd_target_name}")
+    endif()
+    if (NOT GEN_RUN_TARGET_NAME)
+        set(GEN_RUN_TARGET_NAME "run-${cmd_target_name}")
+    endif()
+    string(REPLACE ";" " " cmd_desc "${cmd}")
+
+
+    add_custom_target("${GEN_RUN_TARGET_NAME}"
+                      COMMAND ${cmd}
+                      DEPENDS "${_TEST}"
+                      COMMENT "Run ${cmd_desc}"
+                      WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+                      USES_TERMINAL)
+
+    # A dirty hack to copy the dependent DLLs into the build directory so they
+    # get picked up before the ones in the system directory or wherever.
+    if(WIN32)
+        if (NOT TARGET "${_TEST}-copy-dlls")
+            add_custom_target("${_TEST}-copy-dlls"
+                              COMMAND ${CMAKE_COMMAND}
+                                      -D TARGET_FILE=$<TARGET_FILE:${_TEST}>
+                                      -D TARGET_DIR=$<TARGET_FILE_DIR:${_TEST}>
+                                      -D "SEARCH_PATHS=${dll_search_dirs}"
+                                      -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/copy_reqs.cmake"
+                              COMMENT "Copying DLLs for ${_TEST}")
+        endif()
+        # Avoid executing any run targets before the DLL copying
+        add_dependencies(run-copy-dlls "${_TEST}-copy-dlls")
+        add_dependencies("${GEN_RUN_TARGET_NAME}" run-copy-dlls)
+    endif()
+
+    if (GEN_RUN_TARGET_VAR)
+        set("${GEN_RUN_TARGET_VAR}" "${GEN_RUN_TARGET_NAME}" PARENT_SCOPE)
+    endif()
+endfunction()
+
+# run targets generation
+foreach(test IN LISTS run_tests)
+    set(cmds "${test}")
+
+    # test-specific options
+    foreach(opt_name IN LISTS run_opts)
+        set(opt "${${opt_name}}")
+        list(GET opt 0 test_name)
+        list(GET opt 1 test_opt)
+        list(REMOVE_AT opt 0 1)
+
+        if(test IN_LIST test_name)
+            set(new_cmds)
+            foreach(cmd IN LISTS cmds)
+                foreach(val IN LISTS opt)
+                    list(APPEND new_cmds "${cmd} ${test_opt} ${val}")
+                endforeach()
+            endforeach()
+            set(cmds ${new_cmds})
+        endif()
+    endforeach()
+
+    # terse
+    set(new_cmds)
+    foreach(cmd IN LISTS cmds)
+        foreach(val IN LISTS run_terse)
+            list(APPEND new_cmds "${cmd} ${val}")
+        endforeach()
+    endforeach()
+    set(cmds ${new_cmds})
+
+    # certdir
+    if(test IN_LIST run_certdir_tests)
+        set(new_cmds)
+        foreach(cmd IN LISTS cmds)
+            list(APPEND new_cmds "${cmd} ${run_certdir}")
+        endforeach()
+        set(cmds ${new_cmds})
+    endif()
+
+    # threads
+    set(new_cmds)
+    foreach(cmd IN LISTS cmds)
+        foreach(val IN LISTS run_threads)
+            list(APPEND new_cmds "${cmd} ${val}")
+        endforeach()
+    endforeach()
+    set(cmds ${new_cmds})
+
+    # Run targets
+    foreach(cmd IN LISTS cmds)
+        gen_run_taget("${cmd}" "${test}" VAR run_target_name)
+        add_dependencies(run "${run_target_name}")
+    endforeach()
+
+    # Per-test version target
+    gen_run_taget("${test} -V" "${test}" NAME "run-version-${test}")
+    add_dependencies(run-version "run-version-${test}")
+endforeach()

source/cmake/copy_reqs.cmake

@@ -0,0 +1,18 @@
+include(GetPrerequisites)
+
+get_prerequisites(${TARGET_FILE} prereqs 1 1 "" "${SEARCH_PATHS}" "")
+
+foreach(prereq ${prereqs})
+    # Skipping already copied prerequisites
+    if(EXISTS "${TARGET_DIR}/${prereq}")
+        continue()
+    endif()
+
+    gp_resolve_item("" ${prereq} "" "${SEARCH_PATHS}" prereq_path)
+
+    if(prereq_path)
+        # A hack: we abuse configure_file()'s method of atomically updating
+        # the destination to minimise possible race conditions
+        configure_file("${prereq_path}" "${TARGET_DIR}/${prereq}" COPYONLY)
+    endif()
+endforeach()

source/evp_fetch.c

@@ -183,12 +183,13 @@ void do_fetch(size_t num)
      * to be a multiple of the number of fetch entries therefore at the last
      * iteration we may not check all the algorithms.
      */
+    i = 0;
     do {
         /*
          * If we set a fetch type, always use that
          */
         if (exclusive_fetch_type == FETCH_END) {
-            j = i % array_size;
+            j = i++ % array_size;
             fetch_alg = fetch_entries[j].alg;
             j = fetch_entries[j].ftype;
         } else {
@@ -289,10 +290,12 @@ void do_fetch(size_t num)
 static void
 usage(const char *progname)
 {
-    printf("Usage: %s [-t] [-f TYPE:ALGORITHM]" PQ_USAGE_OPT " threadcount\n"
+    printf("Usage: %s [-t] [-f TYPE:ALGORITHM]" PQ_USAGE_OPT " [-V]"
+           " threadcount\n"
            "-t - terse output\n"
            "-f - fetch only the specified algorithm\n"
            PQ_USAGE_DESC
+           "-V - print version information and exit\n"
            "\nEnvironment variables:\n"
            "  EVP_FETCH_TYPE - if no -f option is provided, fetch only\n"
            "                   the specified TYPE:ALGORITHM\n",
@@ -319,7 +322,7 @@ int main(int argc, char *argv[])
     char *fetch_type = getenv("EVP_FETCH_TYPE");
     int opt;
 
-    while ((opt = getopt(argc, argv, "tf:" PQ_GETOPT)) != -1) {
+    while ((opt = getopt(argc, argv, "tf:" PQ_GETOPT "V")) != -1) {
         switch (opt) {
         case 't':
             terse = 1;
@@ -332,6 +335,9 @@ int main(int argc, char *argv[])
             pq = 1;
             break;
 #endif
+        case 'V':
+            perflib_print_version(basename(argv[0]));
+            return EXIT_SUCCESS;
         default:
             usage(basename(argv[0]));
             return EXIT_FAILURE;