From cc4b43d1a8af3e7a404c42593e570bf9f7cbbd9d Mon Sep 17 00:00:00 2001 From: Alexandr Nedvedicky Date: Sat, 11 Oct 2025 08:38:31 +0200 Subject: [PATCH] provide consistent environement to test SSL libraries with 3rd party tool the idea is to have a script (set of scripts) which build and install environement such we can evaluate SSL performance of different SSL implementations. The initial version supports httpd from apache. The idea is to build desired SSL library and install it to custom location (INSTALL ROOT). Then we build apache and siege client and install it to INSTALL ROOT. The proposed approach uses shell script as it is very lightweight and portable between various *NIX flavours. --- bench-scripts/README.md | 90 +++ bench-scripts/apache_bench.sh | 1048 +++++++++++++++++++++++++++++++++ bench-scripts/common_util.sh | 347 +++++++++++ bench-scripts/nginx_bench.sh | 451 ++++++++++++++ 4 files changed, 1936 insertions(+) create mode 100644 bench-scripts/README.md create mode 100755 bench-scripts/apache_bench.sh create mode 100644 bench-scripts/common_util.sh create mode 100755 bench-scripts/nginx_bench.sh diff --git a/bench-scripts/README.md b/bench-scripts/README.md new file mode 100644 index 00000000..d05ea754 --- /dev/null +++ b/bench-scripts/README.md @@ -0,0 +1,90 @@ +# https performance tests with apache (or nginx) + +The scripts here install and configure desired server with OpenSSL [1], +WolfSSL[2], LibreSSL [3], BoringSSL [4] and aws-lc [5]. Script builds +and installs the library to dedicated directory. Each library is built +with its default options which enable/disable features. The script then +uses siege [6] to measure https performance for each library. There are +no command line options everything is controlled using env. variables: + - `BENCH_INSTALL_ROOT` sets the directory under which the SSL libraries + and tools are installed (`/tmp/branch.binaries` by default) + - `BENCH_WORKSPACE_ROOT` sets the workspace directory where libraries and + tools are compiled. + - `BENCH_MAKE_OPTS` command line options for make(1) command + - `BENCH_RESULTS` directory where to save results + (`$BENCH_INSTALL_ROOT/results` by default) + - `BENCH_HTTPS_PORT` port where https test server should listen to + (4430 by default). + - `BENCH_HTTP_PORT` port where http test server should listen to + (8080 by default) + - `BENCH_TEST_TIME` time to run performance test. default value is + 5 minutes (5M). See option `-t` in siege manual [7] for details. + - `BENCH_HOST` hostname/ip address where server is listening to + (127.0.0.1 by default) + - `BENCH_CERT_SUBJ` set to `/CN=localhost` by default + - `BENCH_CERT_ALT_SUBJ` set to `subjectAltName=DNS:localhost,IP:127.0.0.1` + by default +The siege client runs in benchmark mode (with option `-b`). It is told to fetch +16 files until `BENCH_TEST_TIME` elapses. The file sizes are 64B, 128B, +256B, ... 4MB. + +The libraries the benchmark tests are as follows: + - OpenSSL 3.0, 3.1, ... 3.6, master + - WolfSSL 5.8.2 + - BorinSSL master version + - LibreSSL 4.1.0 + - aws-lc master version + +## Apache + +All tests use version 2.4.65 (except wolfssl which must use 2.4.51, however the +apache still does not work with WolfSSL, issue is still being investigated). +The apache server configuration is identical for all SSL libraries. The apache +server is built with mpm worker, event and pre-fork loadable modules. The test +iterates over three server configurations which each uses particular mpm +module. The modules run with their configuration supplied by apache. + +The configuration for apache server (httpd.conf) is saved along the results +together with configuration for mod\_ssl. The script does not change any +parameters except adjustments of file paths. Everything runs with +default settings which come with apache installation. + +## nginx + +All tests use nginx 1.28 (except WolfSSL which uses 1.24). The +`worker_processes` configuration option is st to auto. +Apart from adjusting paths in nginx.conf the script also sets +option `work_process` to auto. Nginx server configuration is +saved along the results for each test. + +## Build requirements + +Requirements for ubuntu are the following: + - ksh + - gnuplot + - git + - ninja-build + - cmake + - wget + - autoconf + - bzip2 + - libpcre2-dev + - libexpat-dev + - golang-go + - zlib1g-dev + - libtool + - g++ + +[1]: https://www.openssl.org/ + +[2]: https://www.wolfssl.com/ + +[3]: https://www.libressl.org/ + +[4]: https://www.chromium.org/Home/chromium-security/boringssl/ + +[5]: https://aws.amazon.com/security/opensource/cryptography/ + +[6]: https://www.joedog.org/siege-home/ + +[7]: https://www.joedog.org/siege-manual/ diff --git a/bench-scripts/apache_bench.sh b/bench-scripts/apache_bench.sh new file mode 100755 index 00000000..c98d1a2a --- /dev/null +++ b/bench-scripts/apache_bench.sh @@ -0,0 +1,1048 @@ +#!/usr/bin/env ksh +# +# Copyright 2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# + +set -x + +# +# +# make sure to disable firewall +# ufw disable +# it feels like ipv6 loopback traffic is disabled on ubuntu +# + +# +# This is the output of apachectl -V we use to test +# libraries: +# +# Server version: Apache/2.4.65 (Unix) +# Server built: Sep 12 2025 14:49:08 +# Server's Module Magic Number: 20120211:141 +# Server loaded: APR 1.7.6, APR-UTIL 1.6.3, PCRE 10.42 2022-12-11 +# Compiled using: APR 1.7.6, APR-UTIL 1.6.3, PCRE 10.42 2022-12-11 +# Architecture: 64-bit +# Server MPM: event +# threaded: yes (fixed thread count) +# forked: yes (variable process count) +# +# the siege client downloads static files which look as follows +# for all tests: +# 64 Sep 12 14:49 test.txt +# 128 Sep 12 14:49 test_1.txt +# 256 Sep 12 14:49 test_2.txt +# 512 Sep 12 14:49 test_3.txt +# 1.0K Sep 12 14:49 test_4.txt +# 2.0K Sep 12 14:49 test_5.txt +# 4.0K Sep 12 14:49 test_6.txt +# 8.0K Sep 12 14:49 test_7.txt +# 16K Sep 12 14:49 test_8.txt +# 32K Sep 12 14:49 test_9.txt +# 64K Sep 12 14:49 test_10.txt +# 128K Sep 12 14:49 test_11.txt +# 256K Sep 12 14:49 test_12.txt +# 512K Sep 12 14:49 test_13.txt +# 1.0M Sep 12 14:49 test_14.txt +# 2.0M Sep 12 14:49 test_15.txt +# 4.0M Sep 12 14:49 test_16.txt +# + +INSTALL_ROOT=${BENCH_INSTALL_ROOT:-"/tmp/bench.binaries"} +RESULT_DIR=${BENCH_RESULTS:-"${INSTALL_ROOT}/results"} +WORKSPACE_ROOT=${BENCH_WORKSPACE_ROOT:-"/tmp/bench.workspace"} +MAKE_OPTS=${BENCH_MAKE_OPTS} +HTTPS_PORT=${BENCH_HTTPS_PORT:-'4430'} +HTTP_PORT=${BENCH_HTTP_PORT:-'8080'} +CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'} +CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'} +TEST_TIME=${BENCH_TEST_TIME:-'5M'} +HOST=${BENCH_HOST:-'127.0.0.1'} +APACHE_VERSION='2.4.65' + +. ./common_util.sh + +function install_wolfssl_for_apache { + typeset VERSION=$1 + typeset WOLFSSL_TAG="v${VERSION}-stable" + typeset DIRNAME="wolfssl-${VERSION}" + typeset WOLFSSL_WORKSPCE="${WORKSPACE_ROOT}/${DIRNAME}" + typeset WOLFSSL_REPO='https://github.com/wolfSSL/wolfssl' + + if [[ -z ${VERSION} ]] ; then + DIRNAME='wolfssl' + WOLFSSL_WORKSPCE="${WORKSPACE_ROOT}/${DIRNAME}" + fi + mkdir -p ${WOLFSSL_WORKSPCE} + cd ${WOLFSSL_WORKSPCE} + git clone "${WOLFSSL_REPO}" . + if [[ $? -ne 0 ]] ; then + # + # make sure master is up-to date just in + # case we build a master version + # + git checkout master || exit 1 + git pull --rebase || exit 1 + fi + + if [[ -n "${VERSION}" ]] ; then + + git branch -l | grep ${VERSION} + if [[ $? -ne 0 ]] ; then + git checkout tags/${WOLFSSL_TAG} -b wolfssl-${VERSION} || exit 1 + fi + fi + + AUTOCONF_VERSION=2.72 AUTOMAKE_VERSION=1.16 ./autogen.sh || exit 1 + + LDFLAGS="-Wl,-rpath,${INSTALL_ROOT}/${SSL_LIB}/lib" \ + CFLAGS="-DOPENSSL_NO_TLSEXT -I${INSTALL_ROOT}/${SSL_LIB}/include" \ + ./configure --prefix="${INSTALL_ROOT}/${DIRNAME}" \ + --enable-apachehttpd \ + --enable-postauth || exit 1 + + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 +} + +# +# download apr and apr-util and unpack them to apachr-src/srclib directory. The +# unpacked directories must be renamed to basenames (name without version +# string) otherwise apache configure script will complain. +# +function bundle_apr { + typeset VERSION=${APR_VERSION:-1.7.6} + typeset SUFFIX='tar.gz' + typeset BASENAME='apr' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset BUILD_DIR="${BASENAME}-${VERSION}" + typeset DOWNLOAD_URL='https://dlcdn.apache.org/apr' + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + typeset SAVE_CWD=`pwd` + + if [[ ! -f "${WORKSPACE_ROOT}/${DOWNLOAD_FILE}" ]] ; then + wget -O "${WORKSPACE_ROOT}/${DOWNLOAD_FILE}" "${DOWNLOAD_LINK}" || exit 1 + fi + + cd $1 + tar xzf "${WORKSPACE_ROOT}/${DOWNLOAD_FILE}" + mv "${BASENAME}-${VERSION}" "${BASENAME}" || exit 1 + + typeset VERSION="${APRU_VERSION:-1.6.3}" + typeset BASENAME='apr-util' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + if [[ ! -f "${WORKSPACE_ROOT}/${DOWNLOAD_FILE}" ]] ; then + wget -O "${WORKSPACE_ROOT}/${DOWNLOAD_FILE}" "${DOWNLOAD_LINK}" || exit 1 + fi + tar xzf "${WORKSPACE_ROOT}/${DOWNLOAD_FILE}" + mv "${BASENAME}-${VERSION}" "${BASENAME}" || exit 1 + + cd "${SAVE_CWD}" +} + +function install_apache { + typeset VERSION=${APACHE_VERSION:-2.4.65} + typeset SUFFIX='tar.bz2' + typeset BASENAME='httpd' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset BUILD_DIR="${BASENAME}-${VERSION}" + typeset DOWNLOAD_URL="https://archive.apache.org/dist/${BASENAME}" + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + typeset SSL_LIB=$1 + + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + + cd "$WORKSPACE_ROOT" + if [[ ! -f "${DOWNLOAD_FILE}" ]] ; then + wget -O "$DOWNLOAD_FILE" "$DOWNLOAD_LINK" || exit 1 + fi + tar xjf "${DOWNLOAD_FILE}" || exit 1 + bundle_apr "${WORKSPACE_ROOT}/${BUILD_DIR}/srclib" + cd "${BUILD_DIR}" + LDFLAGS="-Wl,-rpath,${INSTALL_ROOT}/${SSL_LIB}/lib" \ + CFLAGS="-I${INSTALL_ROOT}/${SSL_LIB}/include" \ + ./configure --prefix="${INSTALL_ROOT}/${SSL_LIB}" \ + --enable-info \ + --enable-ssl \ + --with-included-apr \ + --enable-mpms-shared=all \ + --with-ssl="${INSTALL_ROOT}/${SSL_LIB}" || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 +} + +function install_apache_boring { + typeset VERSION=${APACHE_VERSION:-2.4.65} + typeset SUFFIX='tar.bz2' + typeset BASENAME='httpd' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset BUILD_DIR="${BASENAME}-${VERSION}" + typeset DOWNLOAD_URL="https://archive.apache.org/dist/${BASENAME}" + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + typeset SSL_LIB=$1 + + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + + cd "$WORKSPACE_ROOT" + if [[ ! -f "${DOWNLOAD_FILE}" ]] ; then + wget -O "$DOWNLOAD_FILE" "$DOWNLOAD_LINK" || exit 1 + fi + tar xjf "${DOWNLOAD_FILE}" || exit 1 + bundle_apr "${WORKSPACE_ROOT}/${BUILD_DIR}/srclib" + # + # do not byild apache benchmark when building with boringssl + # + cd "${BUILD_DIR}" +cat <server); + + #ifdef OPENSSL_NO_OCSP +- if (flag) { + return "OCSP support disabled in SSL library; cannot enable " + "OCSP validation"; +- } + #endif + + return ssl_cmd_ocspcheck_parse(cmd, arg, &sc->server->ocsp_mask); +diff -r -u modules/ssl/ssl_engine_init.c modules/ssl/ssl_engine_init.c +--- modules/ssl/ssl_engine_init.c 2025-07-07 12:09:30.000000000 +0000 ++++ modules/ssl/ssl_engine_init.c 2025-09-25 13:35:31.341000063 +0000 +@@ -1054,15 +1054,6 @@ + ssl_log_ssl_error(SSLLOG_MARK, APLOG_EMERG, s); + return ssl_die(s); + } +-#if SSL_HAVE_PROTOCOL_TLSV1_3 +- if (mctx->auth.tls13_ciphers +- && !SSL_CTX_set_ciphersuites(ctx, mctx->auth.tls13_ciphers)) { +- ap_log_error(APLOG_MARK, APLOG_EMERG, 0, s, APLOGNO(10127) +- "Unable to configure permitted TLSv1.3 ciphers"); +- ssl_log_ssl_error(SSLLOG_MARK, APLOG_EMERG, s); +- return ssl_die(s); +- } +-#endif + return APR_SUCCESS; + } + +@@ -1358,7 +1349,7 @@ + * off the OpenSSL stack and evaluates to true only for the first + * case. With OpenSSL < 3 the second case is identifiable by the + * function code, but function codes are not used from 3.0. */ +-#if OPENSSL_VERSION_NUMBER < 0x30000000L ++#if OPENSSL_VERSION_NUMBER < 0x30000000L && !defined(BORINGSSL_API_VERSION) + #define CHECK_PRIVKEY_ERROR(ec) (ERR_GET_FUNC(ec) != X509_F_X509_CHECK_PRIVATE_KEY) + #else + #define CHECK_PRIVKEY_ERROR(ec) (ERR_GET_LIB(ec) != ERR_LIB_X509 \' +@@ -1580,7 +1571,7 @@ + num_bits, vhost_id, certfile); + } + } +-#if !MODSSL_USE_OPENSSL_PRE_1_1_API ++#if !MODSSL_USE_OPENSSL_PRE_1_1_API && !defined(BORINGSSL_API_VERSION) + if (!custom_dh_done) { + /* If no parameter is manually configured, enable auto + * selection. */ +@@ -1751,7 +1742,7 @@ + + ap_assert(store != NULL); /* safe to assume always non-NULL? */ + +-#if OPENSSL_VERSION_NUMBER >= 0x1010100fL && !defined(LIBRESSL_VERSION_NUMBER) ++#if OPENSSL_VERSION_NUMBER >= 0x1010100fL && !defined(LIBRESSL_VERSION_NUMBER) && !defined(BORINGSSL_API_VERSION) + /* For OpenSSL >=1.1.1, turn on client cert support which is + * otherwise turned off by default (by design). + * https://github.com/openssl/openssl/issues/6933 */ +diff -r -u modules/ssl/ssl_engine_io.c modules/ssl/ssl_engine_io.c +--- modules/ssl/ssl_engine_io.c 2024-07-04 15:58:17.000000000 +0000 ++++ modules/ssl/ssl_engine_io.c 2025-09-25 13:48:43.064886685 +0000 +@@ -234,7 +234,7 @@ + * be expensive in cases where requests/responses are pipelined, + * so limit the performance impact to handshake time. + */ +-#if OPENSSL_VERSION_NUMBER < 0x0009080df ++#if OPENSSL_VERSION_NUMBER < 0x0009080df || defined(BORINGSSL_API_VERSION) + need_flush = !SSL_is_init_finished(outctx->filter_ctx->pssl); + #else + need_flush = SSL_in_connect_init(outctx->filter_ctx->pssl); +@@ -2379,6 +2379,7 @@ + int argi, long argl, long rc) + #endif + { ++#ifndef BORINGSSL_API_VERSION + SSL *ssl; + conn_rec *c; + server_rec *s; +@@ -2453,17 +2454,20 @@ + bio, argp); + } + } ++#endif + return rc; + } + + static APR_INLINE void set_bio_callback(BIO *bio, void *arg) + { ++#ifndef BORINGSSL_API_VERSION + #if OPENSSL_VERSION_NUMBER >= 0x30000000L + BIO_set_callback_ex(bio, modssl_io_cb); + #else + BIO_set_callback(bio, modssl_io_cb); + #endif + BIO_set_callback_arg(bio, arg); ++#endif + } + + void modssl_set_io_callbacks(SSL *ssl, conn_rec *c, server_rec *s) +diff -r -u modules/ssl/ssl_engine_kernel.c modules/ssl/ssl_engine_kernel.c +--- modules/ssl/ssl_engine_kernel.c 2025-07-07 12:09:30.000000000 +0000 ++++ modules/ssl/ssl_engine_kernel.c 2025-09-25 13:35:31.341661132 +0000 +@@ -459,6 +459,7 @@ + const SSL_CIPHER *cipher = NULL; + int depth, verify_old, verify, n, rc; + const char *ncipher_suite; ++ size_t index; + + #ifdef HAVE_SRP + /* +@@ -570,7 +571,7 @@ + renegotiate = TRUE; + } + else if (cipher && cipher_list && +- (sk_SSL_CIPHER_find(cipher_list, cipher) < 0)) ++ (sk_SSL_CIPHER_find(cipher_list, &index, cipher) < 0)) + { + renegotiate = TRUE; + } +@@ -589,7 +590,7 @@ + { + const SSL_CIPHER *value = sk_SSL_CIPHER_value(cipher_list, n); + +- if (sk_SSL_CIPHER_find(cipher_list_old, value) < 0) { ++ if (sk_SSL_CIPHER_find(cipher_list_old, &index, value) < 0) { + renegotiate = TRUE; + } + } +@@ -600,7 +601,7 @@ + { + const SSL_CIPHER *value = sk_SSL_CIPHER_value(cipher_list_old, n); + +- if (sk_SSL_CIPHER_find(cipher_list, value) < 0) { ++ if (sk_SSL_CIPHER_find(cipher_list, &index, value) < 0) { + renegotiate = TRUE; + } + } +@@ -672,7 +673,9 @@ + * are any changes required. + */ + SSL_set_verify(ssl, verify, ssl_callback_SSLVerify); ++#ifndef BORINGSSL_API_VERSION + SSL_set_verify_result(ssl, X509_V_OK); ++#endif + + /* determine whether we've to force a renegotiation */ + if (!renegotiate && verify != verify_old) { +@@ -868,8 +871,9 @@ + "Re-negotiation verification step failed"); + ssl_log_ssl_error(SSLLOG_MARK, APLOG_ERR, r->server); + } +- ++#ifndef BORINGSSL_API_VERSION + SSL_set_verify_result(ssl, X509_STORE_CTX_get_error(cert_store_ctx)); ++#endif + X509_STORE_CTX_cleanup(cert_store_ctx); + X509_STORE_CTX_free(cert_store_ctx); + +@@ -974,7 +978,7 @@ + */ + if (cipher_list) { + cipher = SSL_get_current_cipher(ssl); +- if (sk_SSL_CIPHER_find(cipher_list, cipher) < 0) { ++ if (sk_SSL_CIPHER_find(cipher_list, &index, cipher) < 0) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(02264) + "SSL cipher suite not renegotiated: " + "access to %s denied using cipher %s", +@@ -1096,6 +1100,7 @@ + + SSL_set_verify(ssl, vmode_needed, ssl_callback_SSLVerify); + ++#ifndef BORINGSSL_API_VERSION + if (SSL_verify_client_post_handshake(ssl) != 1) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, APLOGNO(10158) + "cannot perform post-handshake authentication"); +@@ -1105,6 +1110,7 @@ + SSL_set_verify(ssl, vmode_inplace, NULL); + return HTTP_FORBIDDEN; + } ++#endif + + modssl_set_app_data2(ssl, r); + +diff -r -u modules/ssl/ssl_engine_pphrase.c modules/ssl/ssl_engine_pphrase.c +--- modules/ssl/ssl_engine_pphrase.c 2024-11-25 13:37:20.000000000 +0000 ++++ modules/ssl/ssl_engine_pphrase.c 2025-09-25 13:35:31.341966771 +0000 +@@ -30,7 +30,12 @@ + -- Clifford Stoll */ + #include "ssl_private.h" + ++#ifdef BORINGSSL_API_VERSION ++#define PEMerr(...) (void)(0) ++#define EVP_read_pw_string(...) 1 ++#else + #include ++#endif + #if MODSSL_HAVE_OPENSSL_STORE + #include + #endif +diff -r -u modules/ssl/ssl_private.h modules/ssl/ssl_private.h +--- modules/ssl/ssl_private.h 2025-07-07 12:09:30.000000000 +0000 ++++ modules/ssl/ssl_private.h 2025-09-25 13:35:31.342140649 +0000 +@@ -98,7 +98,10 @@ + #include + #include + #include ++#include ++#ifndef OPENSSL_NO_OCSP + #include ++#endif + #include + #if OPENSSL_VERSION_NUMBER >= 0x30000000 + #include +diff -r -u support/Makefile.in support/Makefile.in +--- support/Makefile.in 2018-02-09 10:17:30.000000000 +0000 ++++ support/Makefile.in 2025-09-25 13:35:31.342293168 +0000 +@@ -3,7 +3,7 @@ + + CLEAN_TARGETS = suexec + +-bin_PROGRAMS = htpasswd htdigest htdbm ab logresolve httxt2dbm ++bin_PROGRAMS = htpasswd htdigest htdbm logresolve httxt2dbm + sbin_PROGRAMS = htcacheclean rotatelogs \$(NONPORTABLE_SUPPORT) + TARGETS = \$(bin_PROGRAMS) \$(sbin_PROGRAMS) + +EOF + LDFLAGS="-Wl,-rpath,${INSTALL_ROOT}/${SSL_LIB}/lib -L${INSTALL_ROOT}/${SSL_LIB}/lib -ldecrepit" \ + CFLAGS="-DOPENSSL_NO_TLSEXT -I${INSTALL_ROOT}/${SSL_LIB}/include" \ + ./configure --prefix="${INSTALL_ROOT}/${SSL_LIB}" \ + --enable-info \ + --enable-ssl \ + --with-included-apr \ + --enable-mpms-shared=all \ + --with-ssl="${INSTALL_ROOT}/${SSL_LIB}" || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 +} + +function install_apache_aws { + typeset VERSION=${APACHE_VERSION:-2.4.65} + typeset SUFFIX='tar.bz2' + typeset BASENAME='httpd' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset BUILD_DIR="${BASENAME}-${VERSION}" + typeset DOWNLOAD_URL="https://archive.apache.org/dist/${BASENAME}" + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + typeset SSL_LIB=$1 + + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + + cd "$WORKSPACE_ROOT" + if [[ ! -f "${DOWNLOAD_FILE}" ]] ; then + wget -O "$DOWNLOAD_FILE" "$DOWNLOAD_LINK" || exit 1 + fi + tar xjf "${DOWNLOAD_FILE}" || exit 1 + bundle_apr "${WORKSPACE_ROOT}/${BUILD_DIR}/srclib" + # + # we need to patch apache so it builds with aws. + # this patch seems to be good enough for testing. + # should not be used in production + # + cd "${BUILD_DIR}" +cat <= 0x10101000L && !defined(LIBRESSL_VERSION_NUMBER) + /* +@@ -1358,7 +1360,7 @@ + * off the OpenSSL stack and evaluates to true only for the first + * case. With OpenSSL < 3 the second case is identifiable by the + * function code, but function codes are not used from 3.0. */ +-#if OPENSSL_VERSION_NUMBER < 0x30000000L ++#if OPENSSL_VERSION_NUMBER < 0x30000000L && !defined(OPENSSL_IS_AWSLC) + #define CHECK_PRIVKEY_ERROR(ec) (ERR_GET_FUNC(ec) != X509_F_X509_CHECK_PRIVATE_KEY) + #else + #define CHECK_PRIVKEY_ERROR(ec) (ERR_GET_LIB(ec) != ERR_LIB_X509 \\ +@@ -1751,7 +1753,7 @@ + + ap_assert(store != NULL); /* safe to assume always non-NULL? */ + +-#if OPENSSL_VERSION_NUMBER >= 0x1010100fL && !defined(LIBRESSL_VERSION_NUMBER) ++#if OPENSSL_VERSION_NUMBER >= 0x1010100fL && !(defined(LIBRESSL_VERSION_NUMBER) || defined(OPENSSL_IS_AWSLC)) + /* For OpenSSL >=1.1.1, turn on client cert support which is + * otherwise turned off by default (by design). + * https://github.com/openssl/openssl/issues/6933 */ +EOF + CFLAGS="-DOPENSSL_NO_TLSEXT -I${INSTALL_ROOT}/${SSL_LIB}/include" \ + LDFLAGS="-Wl,-rpath,${INSTALL_ROOT}/${SSL_LIB}/lib" \ + ./configure --prefix="${INSTALL_ROOT}/${SSL_LIB}" \ + --enable-info \ + --enable-ssl \ + --with-included-apr \ + --enable-mpms-shared=all \ + --with-ssl="${INSTALL_ROOT}/${SSL_LIB}" || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 +} + +# +# we need as build dependency for apache +# looks like apache build system not always picks it up +# from system +# +function install_pcre { + typeset SSL_LIB=$1 + typeset VERSION='8.45' + typeset SUFFIX='tar.bz2' + typeset BASENAME='pcre' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset BUILD_DIR="${BASENAME}-${VERSION}" + typeset DOWNLOAD_URL="https://sourceforge.net/projects/pcre/files/pcre/${VERSION}" + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}"/download + + cd "${WORKSPACE_ROOT}" + if [[ -z "${SSL_LIB}" ]] ; then + exit 1 + fi + + if [[ ! -f "${DOWNLOAD_FILE}" ]] ; then + wget -O "${DOWNLOAD_FILE}" "${DOWNLOAD_LINK}" || exit 1 + fi + tar xjf "${DOWNLOAD_FILE}" || exit 1 + cd "${BUILD_DIR}" + ./configure --prefix="${INSTALL_ROOT}/${SSL_LIB}" || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 + cd "${WORKSPACE_ROOT}" +} + +# +# we need a libtool to be able to run buildconf +# +function install_libtool { + typeset SSL_LIB=$1 + typeset VERSION='2.5.4' + typeset SUFFIX='tar.gz' + typeset BASENAME='libtool' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset BUILD_DIR="${BASENAME}-${VERSION}" + typeset DOWNLOAD_URL="https://ftpmirror.gnu.org/libtool/" + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + + if [[ -z "${SSL_LIB}" ]] ; then + exit 1 + fi + + cd "${WORKSPACE_ROOT}" + if [[ ! -f "${DOWNLOAD_FILE}" ]] ; then + wget -O "${DOWNLOAD_FILE}" "${DOWNLOAD_LINK}" || exit 1 + fi + tar xzf "${DOWNLOAD_FILE}" || exit 1 + cd "${BUILD_DIR}" + ./configure --prefix="${INSTALL_ROOT}/${SSL_LIB}" || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 + cd "${WORKSPACE_ROOT}" +} + + +function install_wolf_apache { + typeset VERSION='2.4.51' + typeset SUFFIX='tar.bz2' + typeset BASENAME='httpd' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset BUILD_DIR="${BASENAME}-${VERSION}" + typeset DOWNLOAD_URL="https://archive.apache.org/dist/${BASENAME}" + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + typeset SSL_LIB=$1 + + if [[ -z "${SSL_LIB}" ]] ; then + echo 'ssl library must be specified' + exit 1 + fi + + install_pcre "${SSL_LIB}" + install_libtool "${SSL_LIB}" + + cd "${WORKSPACE_ROOT}" + # + # downgrade apache version for wolf, because + # wolf needs to apply its own set of patches. + # + DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + BUILD_DIR="${BASENAME}-${VERSION}" + if [[ ! -f "${DOWNLOAD_FILE}" ]] ; then + wget -O "$DOWNLOAD_FILE" "$DOWNLOAD_LINK" || exit 1 + fi + tar xjf "${DOWNLOAD_FILE}" || exit 1 + # + # clone wolf's opens source projects (a.k.a. wolf's ports) + # https://github.com/wolfSSL/osp + # we need this to obtain patch for apache sources + # + git clone https://github.com/wolfSSL/osp --depth 1 || exit 1 + cd "${BUILD_DIR}" + patch -p1 < ../osp/apache-httpd/svn_apache-${VERSION}_patch.diff || exit 1 + cd "${WORKSPACE_ROOT}" + bundle_apr "${WORKSPACE_ROOT}/${BUILD_DIR}/srclib" + cd "${BUILD_DIR}" + # + # unlike other ssl implementations wolf requires + # mod_ssl to be linked statically with apache daemon + # + PATH=${PATH}:"${INSTALL_ROOT}/${SSL_LIB}/bin" ./buildconf || exit 1 + ./configure --prefix="${INSTALL_ROOT}/${SSL_LIB}" \ + --enable-info \ + --enable-ssl \ + --with-included-apr \ + --with-pcre="${INSTALL_ROOT}/${SSL_LIB}" \ + --with-wolfssl="${INSTALL_ROOT}/${SSL_LIB}"\ + --enable-mpms-shared=all \ + --with-libxml2 \ + --enable-mods-static=all || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 +} + +function config_apache { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" + typeset HTTPS_CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/extra/httpd-ssl.conf" + typeset SERVERCERT="${INSTALL_ROOT}/${SSL_LIB}/conf/server.crt" + typeset SERVERKEY="${INSTALL_ROOT}/${SSL_LIB}/conf/server.key" + # + # this is hack as we always assume openssl from master version + # is around. We need the tool to create cert and key for server + # + typeset OPENSSL="${INSTALL_ROOT}/openssl-master/bin/openssl" + typeset MOD_SSL="${INSTALL_ROOT}/${SSL_LIB}/modules/mod_ssl.so" + typeset SERVER_NAME="${BENCH_SERVER_NAME:-localhost}" + SERVER_NAME="${SERVER_NAME}:${HTTPS_PORT}" + + # + # enable mod_ssl + # + # we also need slashes in openssl not to confuse sed(1) which adjusts + # apache configuration. + # + typeset SANITZE_SSL=$(echo ${MOD_SSL} | sed -e 's/\//\\\//g') + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e "s/^#\(LoadModule ssl_module\)\(.*$\)/\1 ${SANITZE_SSL}/" \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # load ssl config + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/^#\(Include conf.*httpd-ssl.conf*$\)/\1/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # https 443 will be on 4430 + # + cp "${HTTPS_CONF_FILE}" "${HTTPS_CONF_FILE}".wrk + sed -e "s/^Listen 443.*$/Listen ${HTTPS_PORT}/g" \ + "${HTTPS_CONF_FILE}".wrk > "${HTTPS_CONF_FILE}" || exit 1 + # + # http 80 will be on 8080 + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e "s/^Listen 80.*$/Listen ${HTTP_PORT}/g" \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # load mpm configuration + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^#\)\(Include conf.*httpd-mpm.conf$\)/\2/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # + # fix VirtualHost for 4430 + # + cp "${HTTPS_CONF_FILE}" "${HTTPS_CONF_FILE}".wrk + sed -e "s/\(^\)$/\1${HTTPS_PORT}>/g" \ + "${HTTPS_CONF_FILE}".wrk > "${HTTPS_CONF_FILE}" || exit 1 + + # + # fix ServerName in http.conf + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e "s/^#ServerName.*/ServerName ${SERVER_NAME}/g" \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + # + # fix ServerName + # + cp "${HTTPS_CONF_FILE}" "${HTTPS_CONF_FILE}".wrk + sed -e "s/^ServerName.*/ServerName ${SERVER_NAME}/g" \ + "${HTTPS_CONF_FILE}".wrk > "${HTTPS_CONF_FILE}" || exit 1 + + # + # disable SSLSessionCache, we use worker thread model, if I understand + # documentation right we don't need to share cache between processes. + # + cp "${HTTPS_CONF_FILE}" "${HTTPS_CONF_FILE}".wrk + sed -e 's/\(^SSLSessionCache.*$\)/#\1/g' "${HTTPS_CONF_FILE}".wrk > \ + "${HTTPS_CONF_FILE}" || exit 1 + + gen_certkey $SERVERCERT $SERVERKEY + + generate_download_files "${INSTALL_ROOT}/${SSL_LIB}/htdocs" +} + +function enable_mpm_event { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" + + # + # comment out currently loaded mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # enable event mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^#\)\(LoadModule mpm_event_module .*$\)/\2/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 +} + +function enable_mpm_worker { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" + + # + # comment out currently loaded mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # enable worker mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^#\)\(LoadModule mpm_worker_module .*$\)/\2/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 +} + +function enable_mpm_prefork { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" + + # + # comment out currently loaded mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # enable pre-fork mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^#\)\(LoadModule mpm_prefork_module .*$\)/\2/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 +} + +function run_test { + typeset SSL_LIB=$1 + typeset HTTP='https' + typeset i=0 + typeset PORT=${HTTPS_PORT} + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset RESULTS="${SSL_LIB}".txt + if [[ "${SSL_LIB}" = 'nossl' ]] ; then + HTTP='http' + SSL_LIB='openssl-master' + RESULTS='nossl.txt' + PORT=${HTTP_PORT} + fi + typeset HTDOCS="${INSTALL_ROOT}/${SSL_LIB}"/htdocs + typeset SIEGE="${INSTALL_ROOT}"/openssl-master/bin/siege + + # + # we always try to use siege from openssl master by default, + # if not found then we try the one which is installed for + # openssl version we'd like to test. + # + if [[ ! -x "${SIEGE}" ]] ; then + echo "no ${SIEGE}" + exit 1 + fi + + # + # generate URLs for sewage + # + rm -f siege_urls.txt + for i in `ls -1 ${HTDOCS}/*.txt` ; do + echo "${HTTP}://${HOST}:${PORT}/`basename $i`" >> siege_urls.txt + done + + # + # start apache httpd server + # + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ + ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k start || exit 1 + if [[ $? -ne 0 ]] ; then + echo "could not start ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd" + exit 1 + fi + LD_LIBRARY_PATH=${INSTALL_ROOT}/openssl-master/lib "${SIEGE}" -t ${TEST_TIME} -b \ + -f siege_urls.txt 2> "${RESULT_DIR}/${RESULTS}" + if [[ $? -ne 0 ]] ; then + echo "${INSTALL_ROOT}/${SSL_LIB} can not run siege" + cat "${RESULT_DIR}/${RESULTS}" + exit 1 + fi + + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ + ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k stop || exit 1 + pgrep httpd + while [[ $? -eq 0 ]] ; do + sleep 1 + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ + ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k stop || exit 1 + echo "stopping ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd" + pgrep httpd + done + + # + # save apache configuration used for testing along the results. + # we do care about httpd.conf and httpd-ssl.conf only as only + # those two were modified. + # + cp ${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf \ + ${RESULT_DIR}/httpd-${SSL_LIB}.conf + cp ${INSTALL_ROOT}/${SSL_LIB}/conf/extra/httpd-ssl.conf \ + ${RESULT_DIR}/httpd-ssl-${SSL_LIB}.conf +} + +function setup_tests { + install_openssl master + install_siege openssl-master + install_apache openssl-master + config_apache openssl-master + cd "${WORKSPACE_ROOT}" + clean_build + + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + install_openssl openssl-$i ; + install_apache openssl-$i + config_apache openssl-$i + cd "${WORKSPACE_ROOT}" + clean_build + done + + install_openssl OpenSSL_1_1_1-stable + install_apache OpenSSL_1_1_1-stable + config_apache OpenSSL_1_1_1-stable + cd "${WORKSPACE_ROOT}" + clean_build + + # + # wolf-ssl does not work. It installs it starts, + # client can establish connection but handshake + # seems to get stuck. I can see client sends its + # hello with TLS-1.2/TLS-1.3 and there is no + # reply from server, for more than 10secs. + # + # this is the configuration I'm using: + # ServerName localhost + # Listen 4430 + # + # SSLCipherSuite HIGH:MEDIUM:!MD5:!RC4:!3DES + # SSLHonorCipherOrder on + # SSLProtocol all -SSLv3 + # + # ServerName localhost + # DocumentRoot "/home/sashan/work.openssl/bench.binaries/wolfssl-5.8.2/htdocs" + # ErrorLog "/home/sashan/work.openssl/bench.binaries/wolfssl-5.8.2/logs/error_log" + # TransferLog "/home/sashan/work.openssl/bench.binaries/wolfssl-5.8.2/logs/access_log" + # SSLEngine on + # SSLCertificateFile "/home/sashan/work.openssl/bench.binaries/wolfssl-5.8.2/conf/server.crt" + # SSLCertificateKeyFile "/home/sashan/work.openssl/bench.binaries/wolfssl-5.8.2/conf/server.key" + # + # + # Unlike the suggested configuration here: + # https://github.com/wolfSSL/osp/blob/master/apache-httpd/README.md#running-simple-https + # I had to add SSLCipherSuite SSLHonorCipherOrder SSLProtocol. The configuration from + # link above does not work either. The httpd process refuses to start, leaving message: + # + # [Fri Aug 29 17:10:39.065428 2025] [ssl:emerg] [pid 3263901:tid 133639498441152] AH01898: Unable to configure permitted SSL ciphers + # [Fri Aug 29 17:10:39.065460 2025] [ssl:emerg] [pid 3263901:tid 133639498441152] AH02311: Fatal error initialising mod_ssl, exiting. See /home/sashan/work.openssl/bench.binaries/wolfssl-5.8.2/logs/error_log for more information + # AH00016: Configuration Failed + # + # any hints/advise on how to get apache with wolfssl going is welcomed + # + install_wolfssl_for_apache 5.8.2 + install_wolf_apache wolfssl-5.8.2 + config_apache wolfssl-5.8.2 + cd "${WORKSPACE_ROOT}" + clean_build + + install_libressl 4.1.0 + install_apache libressl-4.1.0 + config_apache libressl-4.1.0 + cd "${WORKSPACE_ROOT}" + clean_build + + install_boringssl + install_apache_boring boringssl + config_apache boringssl + cd "${WORKSPACE_ROOT}" + clean_build + + install_aws_lc + install_apache_aws aws-lc + config_apache aws-lc + cd "${WORKSPACE_ROOT}" + clean_build +} + +function run_tests { + typeset SAVE_RESULT_DIR="${RESULT_DIR}" + + for i in event worker pre-fork ; do + mkdir -p ${RESULT_DIR}/$i || exit 1 + done + + enable_mpm_event + RESULT_DIR="${SAVE_RESULT_DIR}/event" + run_test nossl + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + enable_mpm_event openssl-${i} + run_test openssl-${i} + done + enable_mpm_event openssl-master + run_test openssl-master + enable_mpm_event OpenSSL_1_1_1-stable + run_test OpenSSL_1_1_1-stable + enable_mpm_event libressl-4.1.0 + run_test libressl-4.1.0 + #enable_mpm_event wolfssl-5.8.2 + #run_test wolfssl-5.8.2 + enable_mpm_event boringssl + run_test boringssl + enable_mpm_event aws-lc + run_test aws-lc + + enable_mpm_worker + RESULT_DIR="${SAVE_RESULT_DIR}/worker" + run_test nossl + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + enable_mpm_worker openssl-${i} + run_test openssl-${i} + done + enable_mpm_worker openssl-master + run_test openssl-master + enable_mpm_worker OpenSSL_1_1_1-stable + run_test OpenSSL_1_1_1-stable + enable_mpm_worker libressl-4.1.0 + run_test libressl-4.1.0 + #enable_mpm_worker wolfssl-5.8.2 + #run_test wolfssl-5.8.2 + enable_mpm_worker boringssl + run_test boringssl + enable_mpm_worker aws-lc + run_test aws-lc + + enable_mpm_prefork + RESULT_DIR="${SAVE_RESULT_DIR}/pre-fork" + run_test nossl + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + enable_mpm_prefork openssl-${i} + run_test openssl-${i} + done + enable_mpm_prefork openssl-master + run_test openssl-master + enable_mpm_prefork OpenSSL_1_1_1-stable + run_test OpenSSL_1_1_1-stable + enable_mpm_prefork libressl-4.1.0 + run_test libressl-4.1.0 + #enable_mpm_prefork wolfssl-5.8.2 + #run_test wolfssl-5.8.2 + enable_mpm_prefork boringssl + run_test boringssl + enable_mpm_prefork aws-lc + run_test aws-lc + + RESULT_DIR=${SAVE_RESULT_DIR} +} + +check_env +setup_tests +run_tests +SAVE_RESULT_DIR=${RESULT_DIR} +for i in event worker pre-fork ; do + RESULT_DIR=${SAVE_RESULT_DIR}/$i + plot_results +done +RESULT_DIR=${SAVE_RESULT_DIR} + +echo "testing using siege is complete, results can be found ${RESULT_DIR}:" diff --git a/bench-scripts/common_util.sh b/bench-scripts/common_util.sh new file mode 100644 index 00000000..21f2230d --- /dev/null +++ b/bench-scripts/common_util.sh @@ -0,0 +1,347 @@ +#!/usr/bin/env ksh +# +# Copyright 2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# + +set -x + +function check_env { + if [[ ! -x "$(which gnuplot)" ]] ; then + echo 'No gnuplot in PATH' + exit 1 + fi + + if [[ ! -x "$(which git)" ]] ; then + echo 'No git in PATH' + exit 1 + fi + + if [[ ! -x "$(which ninja)" ]] ; then + echo "No ninja in PATH" + exit 1 + fi + + if [[ ! -x "$(which cmake)" ]] ; then + echo 'No cmake in PATH' + exit 1 + fi + + if [[ ! -x "$(which wget)" ]] ; then + echo 'No wget in PATH' + exit 1 + fi + + if [[ ! -x "$(which autoconf)" ]] ; then + echo 'No autoconf in PATH' + exit 1 + fi + + if [[ ! -x "$(which automake)" ]] ; then + echo 'No automake in PATH' + exit 1 + fi + + if [[ ! -x "$(which seq)" ]] ; then + echo 'No seq in PATH' + exit 1 + fi + + typeset TEST_FILE=".test_file.$$" + mkdir -p "${WORKSPACE_ROOT}" + if [[ $? -ne 0 ]] ; then + echo "Can not create ${WORKSPACE_ROOT}" + exit 1; + fi + touch "${WORKSPACE_ROOT}/${TEST_FILE}" + if [[ $? -ne 0 ]] ; then + echo "${WORKSPACE_ROOT} is not writable" + exit 1 + fi + + mkdir -p "${INSTALL_ROOT}" + if [[ $? -ne 0 ]] ; then + echo "Can not create ${INSTALL_ROOT}" + exit 1; + fi + touch "${INSTALL_ROOT}/${TEST_FILE}" + if [[ $? -ne 0 ]] ; then + echo "${INSTALL_ROOT} is not writable" + exit 1 + fi + + mkdir -p "${RESULT_DIR}" + touch "${RESULT_DIR}/${TEST_FILE}" + if [[ $? -ne 0 ]] ; then + echo "${RESULT_DIR} is not writable" + exit 1 + fi + + rm -f "${INSTALL_ROOT}/${TEST_FILE}" + rm -f "${WORKSPACE_ROOT}/${TEST_FILE}" +} + +function cleanup { + rm -rf ${INSTALL_ROOT} + rm -rf ${WORKSPACE_ROOT} +} + +function clean_build { + typeset SAVE_DIR=`pwd` + cd "${WORKSPACE_ROOT}" + for i in * ; do + if [[ -d $i ]] ; then + rm -rf $i + fi + done + cd "${SAVE_DIR}" +} + +function install_openssl { + typeset OPENSSL_REPO='https://github.com/openssl/openssl' + typeset BRANCH_NAME=$1 + typeset DIRNAME='' + + if [[ "${BRANCH_NAME}" = 'master' ]] ; then + DIRNAME='openssl-master' + else + DIRNAME="${BRANCH_NAME}" + fi + + cd ${WORKSPACE_ROOT} + mkdir -p ${DIRNAME} + cd ${DIRNAME} + + git clone --single-branch -b ${BRANCH_NAME} --depth 1 \ + "${OPENSSL_REPO}" . || exit 1 + ./config --prefix="${INSTALL_ROOT}/${DIRNAME}" \ + --libdir="lib" || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 +} + +function install_wolfssl { + typeset VERSION=$1 + typeset WOLFSSL_TAG="v${VERSION}-stable" + typeset DIRNAME="wolfssl-${VERSION}" + typeset WOLFSSL_WORKSPCE="${WORKSPACE_ROOT}/${DIRNAME}" + typeset WOLFSSL_REPO='https://github.com/wolfSSL/wolfssl' + + if [[ -z ${VERSION} ]] ; then + DIRNAME='wolfssl' + WOLFSSL_WORKSPCE="${WORKSPACE_ROOT}/${DIRNAME}" + fi + mkdir -p ${WOLFSSL_WORKSPCE} + cd ${WOLFSSL_WORKSPCE} + git clone "${WOLFSSL_REPO}" . + if [[ $? -ne 0 ]] ; then + # + # make sure master is up-to date just in + # case we build a master version + # + git checkout master || exit 1 + git pull --rebase || exit 1 + fi + + if [[ -n "${VERSION}" ]] ; then + + git branch -l | grep ${VERSION} + if [[ $? -ne 0 ]] ; then + git checkout tags/${WOLFSSL_TAG} -b wolfssl-${VERSION} || exit 1 + fi + fi + + AUTOCONF_VERSION=2.72 AUTOMAKE_VERSION=1.16 ./autogen.sh || exit 1 + + ./configure --prefix="${INSTALL_ROOT}/${DIRNAME}" \ + --enable-nginx || exit 1 + + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 +} + +function install_libressl { + typeset VERSION=${1:-4.1.0} + typeset SUFFIX='tar.gz' + typeset BASENAME='libressl' + typeset DOWNLOAD_FILE="${BASENAME}-${VERSION}.${SUFFIX}" + typeset BUILD_DIR="${BASENAME}-${VERSION}" + typeset DOWNLOAD_URL='https://cdn.openbsd.org/pub/OpenBSD/LibreSSL/' + typeset DOWNLOAD_LINK="${DOWNLOAD_URL}/${DOWNLOAD_FILE}" + + cd "$WORKSPACE_ROOT" + if [[ ! -f "${DOWNLOAD_FILE}" ]] ; then + wget -O "$DOWNLOAD_FILE" "$DOWNLOAD_LINK" || exit 1 + fi + tar xzf "${DOWNLOAD_FILE}" + cd ${BUILD_DIR} + ./configure --prefix="${INSTALL_ROOT}/${BUILD_DIR}" || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 +} + +function install_boringssl { + typeset BORING_REPO='https://boringssl.googlesource.com/boringssl' + typeset BORING_NAME='boringssl' + cd "${WORKSPACE_ROOT}" + mkdir -p "${BORING_NAME}" + cd "${BORING_NAME}" + git clone "${BORING_REPO}" --depth 1 . || exit 1 + # + # we need to install libdecrepit.so so mod_ssl can use + # base64 BIO file stream + # +cat < ${DATA_FILE} + cd "${RESULT_DIR}" + for LIBRARY in `ls *.txt |sed -e 's/\.txt$//g'` ; do + RESULT_FILE="${RESULT_DIR}/${LIBRARY}.txt" + VALUE=`grep "^${MATCH}" ${RESULT_FILE} | cut -f 2 -d : | awk '{ print($1); }'` + echo "${COUNT} ${LIBRARY} ${VALUE}" >> ${DATA_FILE} + COUNT=$((COUNT + 1)) + done +cat < ${PLOT_FILE} +set style fill solid border -1 +set term png size 1600, 600 +set boxwidth 0.4 +set autoscale +set output "${PNG_FILE}" +plot "${DATA_FILE}" using 1:3:xtic(2) with boxes +EOF + gnuplot ${PLOT_FILE} +} + +function plot_results { + plot_chart 'transactions' 'Transactions Total' 'Transactions:' + plot_chart 'data_transferred' 'Data transferred' 'Data transferred:' + plot_chart 'response_time' 'Avg. response time' 'Response time:' + plot_chart 'transaction_rate' 'Transaction Rate' 'Transaction rate:' + plot_chart 'throughput' 'Throughput' 'Throughput:' + plot_chart 'concurrency' 'Concurrency' 'Concurrency:' +} + +function generate_download_files { + typeset HTDOCS=$1 + + mkdir -p ${HTDOCS} || exit 1 + + # + # we start with 64 bytes long file + # + for i in `seq 16` ; do + echo -n 'test' >> "${HTDOCS}"/test.txt + done + + # + # here we double the size of last file with each + # iteration. starting at 64, then 128, 254, 512,... + # + typeset LAST="${HTDOCS}"/test.txt + for i in `seq 16` ; do + cat "${LAST}" "${LAST}" > "${HTDOCS}/test_${i}.txt" + LAST="${HTDOCS}/test_${i}.txt" + done +} + +function gen_certkey { + typeset SERVERCERT=$1 + typeset SERVERKEY=$2 + typeset OPENSSL="${INSTALL_ROOT}"/openssl-master/bin/openssl + + # + # generate self-signed cert with key + # note this is hack because we always assume + # openssl-master is installed in INSTALL root + # + $(LD_LIBRARY_PATH="${INSTALL_ROOT}/openssl-master/lib" "${OPENSSL}" \ + req -x509 -newkey rsa:4096 -days 180 -noenc -keyout \ + "${SERVERKEY}" -out "${SERVERCERT}" -subj "${CERT_SUBJ}" \ + -addext "${CERT_ALT_SUBJ}") || exit 1 +} + diff --git a/bench-scripts/nginx_bench.sh b/bench-scripts/nginx_bench.sh new file mode 100755 index 00000000..394e0772 --- /dev/null +++ b/bench-scripts/nginx_bench.sh @@ -0,0 +1,451 @@ +#!/usr/bin/env ksh +# +# Copyright 2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# + +set -x + +# +# +# make sure to disable firewall +# ufw disable +# it feels like ipv6 loopback traffic is disabled on ubuntu +# +INSTALL_ROOT=${BENCH_INSTALL_ROOT:-"/tmp/bench.binaries"} +RESULT_DIR=${BENCH_RESULTS:-"${INSTALL_ROOT}/results"} +WORKSPACE_ROOT=${BENCH_WORKSPACE_ROOT:-"/tmp/bench.workspace"} +MAKE_OPTS=${BENCH_MAKE_OPTS} +HTTPS_PORT=${BENCH_HTTPS_PORT:-'4430'} +HTTP_PORT=${BENCH_HTTP_PORT:-'8080'} +CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'} +CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'} +TEST_TIME=${BENCH_TEST_TIME:-'5M'} +HOST=${BENCH_HOST:-'127.0.0.1'} + +. ./common_util.sh + +# +# the script builds various libssl libraries: +# openssl, wolfssl, boringss, libressl +# each library is installed to its own install root under INSTALL_ROOT +# directory. The script also builds nginx server version 1.28 and installs +# it alongside each openssl library. +# +# for openssl the build process is straightforward: +# clone desired version from github +# +# build it with prefix set to INSTALL_ROOT/openssl-version +# and install it +# +# then clone the nginx server version 1.28, build process +# for nginx is straightforward. the build options are as follows: +# --with-http_ssl_module \ +# --with-threads \ +# --with-cc-opt="-fPIC" \ +# --with-ld-opt="-Wl,-rpath,${INSTALL_ROOT}/${SSL_LIB}/lib -L ${INSTALL_ROOT}/${SSL_LIB}/lib -lcrypto -L ${INSTALL_ROOT}/${SSL_LIB}/lib -lssl" \ +# --with-openssl="${WORKSPACE_ROOT}/${SSL_LIB}" || exit 1 +# with-openssl flag points to openssl sources +# we deliberately pass lcrypto/lssl as we want to link them dynamically +# without this hack the build process picks static version +# found in WORKSPACE_ROOT/SSL_LIB +# +# for libressl the build process is similar. however we download +# ,tar.gz package instead doing git clone +# +# for boringssl the build process slightly differs as boringssl uses cmake. +# the build flags for boringssl are as follows: +# -DCMAKE_INSTALL_PREFIX="${INSTALL_ROOT}/${BORING_NAME}" \ +# -DBUILD_SHARED_LIBS=1 \ +# -DCMAKE_BUILD_TYPE=Release +# nginx build process needs to be adjusted too. It happens in separate +# function setup_sslib_for_nginx() here in shell. Basically we create +# .openssl directory under $WORKSPACE_ROOT/boringssl sourcetree and populate +# it with boringssl headers files and create ,openssl/lib/libcrypto.a +# and .openssl/lib/libssl.a empty files using touch(1) this hack +# is sufficient to get nginx build process going. The nginx expects +# static libraries but we are forcing it to use dynamic versions +# by tweaking --with-ld-opt flags. +# +# there is a separate function install_wolf_nginx() which builds nginx +# with wolfssl it follows guide found here: +# https://github.com/wolfssl/wolfssl-nginx +# +# all nginx servers use the same configuration: +# +# worker_processes auto; +# events { +# worker_connections 1024; +#} +# +# http { +# include mime.types; +# default_type application/octet-stream; +# #access_log logs/access.log main; +# sendfile on; +# #tcp_nopush on; +# #keepalive_timeout 0; +# keepalive_timeout 65; +# +# #gzip on; +# +# server { +# listen ${HTTP_PORT}; +# server_name ${SERVER_NAME}; +# +# location / { +# root html; +# index index.html index.htm; +# } +# +# #error_page 404 /404.html; +# +# # redirect server error pages to the static page /50x.html +# # +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# root html; +# } +# +# } +# +# # HTTPS server +# # +# server { +# listen ${HTTPS_PORT} ssl; +# server_name ${SERVER_NAME}; +# +# ssl_certificate ${SERVERCERT}; +# ssl_certificate_key ${SERVERKEY}; +# +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# +# location / { +# root html; +# index index.html index.htm; +# } +# } +# } +# +# the serverkey and servercert are self-signed certificate for +# localhost/127.0.0.1 +# +# The performance is tested using siege (https://github.com/JoeDog/siege +# scripts build it and installs it along openssl-master +# the client by default fetches set of 17 urls for 5 minutes to +# gather performance data for each nginx/ssl combination. +# the sizes of files which are downloaded are {64, 128, 256, ... 4MB) +# + + +function setup_sslib_for_nginx { + typeset SSLIB_NAME=$1 + + if [[ -z ${SSLIB_NAME} ]] ; then + exit 1 + fi + + cd "${WORKSPACE_ROOT}" + cd "${SSLIB_NAME}" + # + # based on notes I've found here: + # https://lvv.me/posts/2019/01/24-build_nginx_with_boringssl/ + # but we don't' need to build everything again, we just re-use + # bits from build directory we created library install step. + # + mkdir -p .openssl/lib + # + # this is a hack nginx wants to link with static library, + # however we will be using dynamic library .so + # + touch .openssl/lib/libcrypto.a + touch .openssl/lib/libssl.a + cd .openssl || exit 1 + ln -s ../include . + cd "${WORKSPACE_ROOT}" +} + +function config_nginx { + # + # this is hack as we always assume openssl from master version + # is around. We need the tool to create cert and key for server + # + typeset SSL_LIB=$1 + if [[ -z $SSL_LIB ]] ; then + SSL_LIB='openssl-master' + fi + typeset CONF="${INSTALL_ROOT}/${SSL_LIB}"/conf/nginx.conf + typeset SERVERCERT="${INSTALL_ROOT}/${SSL_LIB}/conf/server.crt" + typeset SERVERKEY="${INSTALL_ROOT}/${SSL_LIB}/conf/server.key" + typeset SERVER_NAME="${BENCH_SERVER_NAME:-localhost}" + +cat < "${CONF}" +worker_processes auto; +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + #access_log logs/access.log main; + sendfile on; + #tcp_nopush on; + #keepalive_timeout 0; + keepalive_timeout 65; + + #gzip on; + + server { + listen ${HTTP_PORT}; + server_name ${SERVER_NAME}; + + location / { + root html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root html; + } + + } + + # HTTPS server + # + server { + listen ${HTTPS_PORT} ssl; + server_name ${SERVER_NAME}; + + ssl_certificate ${SERVERCERT}; + ssl_certificate_key ${SERVERKEY}; + + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; + + location / { + root html; + index index.html index.htm; + } + } +} + + +EOF + gen_certkey $SERVERCERT $SERVERKEY + + generate_download_files "${INSTALL_ROOT}/${SSL_LIB}/html" +} + +function install_nginx { + typeset SSL_LIB=$1 + typeset NGIX_REPO='https://github.com/nginx/nginx' + typeset VERSION='1.28' + typeset BASENAME='nginx' + typeset DIRNAME="${BASENAME}-${VERSION}" + + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + + cd "${WORKSPACE_ROOT}" + mkdir -p "${DIRNAME}" + cd "${DIRNAME}" + git clone "${NGIX_REPO}" -b stable-${VERSION} --depth 1 . || exit 1 + + # + # note ngix unlike apache requires pointer to ssl sources + # also we add .so versions as linker parameter. + # + ./auto/configure --prefix="${INSTALL_ROOT}/${SSL_LIB}" \ + --with-http_ssl_module \ + --with-threads \ + --with-cc-opt="-fPIC" \ + --with-ld-opt="-Wl,-rpath,${INSTALL_ROOT}/${SSL_LIB}/lib -L ${INSTALL_ROOT}/${SSL_LIB}/lib -lcrypto -L ${INSTALL_ROOT}/${SSL_LIB}/lib -lssl" \ + --with-openssl="${WORKSPACE_ROOT}/${SSL_LIB}" || exit 1 + + # + # this is required by boring/aws-lc. it does not hurt wolf/openssl/libre + # comes from here: + # https://lvv.me/posts/2019/01/24-build_nginx_with_boringssl/ + # + touch ../${SSL_LIB}/.openssl/include/openssl/ssl.h + chmod +x ${INSTALL_ROOT}/${SSL_LIB}/lib/*.so + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 + cd "${WORKSPACE_ROOT}" +} + +function install_wolf_nginx { + typeset SSL_LIB=$1 + typeset NGIX_REPO='https://github.com/nginx/nginx' + typeset VERSION='1.24' + typeset BASENAME='nginx' + typeset DIRNAME="${BASENAME}-${VERSION}" + + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + + cd "${WORKSPACE_ROOT}" + mkdir -p "${DIRNAME}" + cd "${DIRNAME}" + git clone "${NGIX_REPO}" -b stable-${VERSION} --depth 1 . || exit 1 + + cd "${WORKSPACE_ROOT}" + git clone https://github.com/wolfssl/wolfssl-nginx --depth 1 || exit 1 + cd "${DIRNAME}" + patch -p1 < ../wolfssl-nginx/nginx-${VERSION}.0-wolfssl.patch || exit 1 + + # + # note nginx unlike apache requires pointer to ssl sources + # + ./auto/configure --prefix="${INSTALL_ROOT}/${SSL_LIB}" \ + --with-http_ssl_module \ + --with-threads \ + --with-ld-opt="-Wl,-rpath,${INSTALL_ROOT}/${SSL_LIB}/lib" \ + --with-wolfssl="${INSTALL_ROOT}/${SSL_LIB}" || exit 1 + make ${MAKE_OPTS} || exit 1 + make ${MAKE_OPTS} install || exit 1 + cd "${WORKSPACE_ROOT}" +} + +function setup_tests { + install_openssl master + install_nginx openssl-master + install_siege openssl-master + config_nginx openssl-master + + cd "${WORKSPACE_ROOT}" + clean_build + + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + install_openssl openssl-$i + install_nginx openssl-$i + config_nginx openssl-$i + cd "${WORKSPACE_ROOT}" + rm -rf * + done + + install_openssl OpenSSL_1_1_1-stable + install_nginx OpenSSL_1_1_1-stable + config_nginx OpenSSL_1_1_1-stable + + install_wolfssl 5.8.2 + install_wolf_nginx wolfssl-5.8.2 + config_nginx wolfssl-5.8.2 + cd "${WORKSPACE_ROOT}" + clean_build + + install_libressl 4.1.0 + install_nginx libressl-4.1.0 + config_nginx libressl-4.1.0 + cd "${WORKSPACE_ROOT}" + clean_build + + install_boringssl + setup_sslib_for_nginx boringssl + install_nginx boringssl + config_nginx boringssl + cd "${WORKSPACE_ROOT}" + clean_build + + +# +# nginx currently does not support aws-lc +# install_aws_lc +# setup_sslib_for_nginx aws-lc +# install_nginx aws-lc +# cd "${WORKSPACE_ROOT}" +# clean_build +} + +function run_test { + typeset SSL_LIB=$1 + typeset HTTP='https' + typeset i=0 + typeset PORT=${HTTPS_PORT} + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset RESULTS="${SSL_LIB}".txt + if [[ "${SSL_LIB}" = 'nossl' ]] ; then + HTTP='http' + SSL_LIB='openssl-master' + RESULTS='nossl.txt' + PORT=${HTTP_PORT} + fi + typeset HTDOCS="${INSTALL_ROOT}/${SSL_LIB}"/html + typeset SIEGE="${INSTALL_ROOT}"/openssl-master/bin/siege + + # + # we always try to use siege from openssl master by default, + # if not found then we try the one which is installed for + # openssl version we'd like to test. + # + if [[ ! -x "${SIEGE}" ]] ; then + echo "no ${SIEGE}" + exit 1 + fi + + rm -f siege_urls.txt + for i in `ls -1 ${HTDOCS}/*.txt` ; do + echo "${HTTP}://${HOST}:${PORT}/`basename $i`" >> siege_urls.txt + done + + # + # start nginx server + # + echo LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx + if [[ $? -ne 0 ]] ; then + echo "could not start ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx" + exit 1 + fi + + LD_LIBRARY_PATH=${INSTALL_ROOT}/openssl-master/lib "${SIEGE}" -t ${TEST_TIME} -b \ + -f siege_urls.txt 2> "${RESULT_DIR}/${RESULTS}" + + # + # stop nginx server + # + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx -s quit + + # + # save nginx.conf used for testing along the results + # + cp ${INSTALL_ROOT}/${SSL_LIB}/conf/nginx.conf ${RESULT_DIR}/nginx-${SSL_LIB}.conf +} + +function run_tests { + run_test nossl + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + run_test openssl-${i} + done + run_test openssl-master + run_test OpenSSL_1_1_1-stable + run_test libressl-4.1.0 + # + # could not get apache with wolfssl working + # + run_test wolfssl-5.8.2 + run_test boringssl + #run_test aws-lc +} + +check_env +setup_tests +run_tests +plot_results + +echo "testing using siege is complete, results can be found ${RESULT_DIR}:"