Github workflow testing support with WOLFPROV_FORCE_FAIL=1

.github/scripts/check-workflow-result.sh

@@ -1,4 +1,23 @@
 #!/bin/bash
+# check-workflow-result.sh
+#
+# Copyright (C) 2006-2024 wolfSSL Inc.
+#
+# This file is part of wolfProvider.
+#
+# wolfProvider is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# wolfProvider is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
 
 set -e
 
@@ -11,14 +30,17 @@ TEST_RESULT="$1"
 WOLFPROV_FORCE_FAIL="${2:-}"
 TEST_SUITE="${3:-}"
 
+# Ensure TEST_RESULT is treated as a number
+TEST_RESULT=$((TEST_RESULT + 0))
+
 # If force fail is empty treat second arg as test suite
 if [ -z "$WOLFPROV_FORCE_FAIL" ]; then
     TEST_SUITE="${2:-}"
 fi
 
 if [ "$WOLFPROV_FORCE_FAIL" = "WOLFPROV_FORCE_FAIL=1" ]; then
+    # ----- CURL -----
     if [ "$TEST_SUITE" = "curl" ]; then
-        # --- curl-specific logic ---
         if [ -f "tests/test.log" ]; then
             # Extract and clean the failed test list from the log
             ACTUAL_FAILS=$(grep -a '^TESTFAIL: These test cases failed:' tests/test.log | sed 's/.*failed: //')
@@ -74,13 +96,128 @@ if [ "$WOLFPROV_FORCE_FAIL" = "WOLFPROV_FORCE_FAIL=1" ]; then
             echo "FAIL: Actual failed tests do not match expected."
             exit 1
         fi
+    # ----- OPENVPN -----
+    elif [ "$TEST_SUITE" = "openvpn" ]; then
+        if [ -f "openvpn-test.log" ]; then
+            # Extract failed tests from the log
+            ACTUAL_FAILS=$(grep -a '^FAIL: ' openvpn-test.log | sed 's/^FAIL: //' | sort)
+
+            # Define expected failures
+            EXPECTED_FAILS="auth_token_testdriver crypto_testdriver pkt_testdriver tls_crypt_testdriver"
+
+            # Create temporary files for sorted lists
+            TEMP_DIR=$(mktemp -d)
+            ACTUAL_SORTED="${TEMP_DIR}/actual_sorted.txt"
+            EXPECTED_SORTED="${TEMP_DIR}/expected_sorted.txt"
+
+            # Clean and sort both lists
+            echo "$ACTUAL_FAILS" | tr ' ' '\n' | grep -v '^$' | sort > "$ACTUAL_SORTED"
+            echo "$EXPECTED_FAILS" | tr ' ' '\n' | grep -v '^$' | sort > "$EXPECTED_SORTED"
+
+            echo "DEBUG: Actual failed tests: $(tr '\n' ' ' < "$ACTUAL_SORTED")"
+            echo "DEBUG: Expected failed tests: $(tr '\n' ' ' < "$EXPECTED_SORTED")"
+
+            # Find missing in actual (in expected but not in actual)
+            MISSING=$(comm -23 "$EXPECTED_SORTED" "$ACTUAL_SORTED" | tr '\n' ' ')
+            # Find extra in actual (in actual but not in expected)
+            EXTRA=$(comm -13 "$EXPECTED_SORTED" "$ACTUAL_SORTED" | tr '\n' ' ')
+
+            # Clean up temporary files
+            rm -rf "$TEMP_DIR"
+
+            echo "Test(s) that should have failed: $MISSING"
+            echo "Test(s) that shouldn't have failed: $EXTRA"
+
+            if [ -z "$MISSING" ] && [ -z "$EXTRA" ]; then
+                echo "PASS: Actual failed tests match expected."
+                exit 0
+            else
+                echo "FAIL: Actual failed tests do not match expected."
+                exit 1
+            fi
+        else
+            echo "Error: openvpn-test.log not found"
+            exit 1
+        fi
+    # ----- SSSD -----
+    elif [ "$TEST_SUITE" = "sssd" ]; then
+        if [ -f "sssd-test.log" ]; then
+            # Extract failed tests from the log
+            ACTUAL_FAILS=$(grep -a '^FAIL: ' sssd-test.log | sed 's/^FAIL: //' | sort)
+
+            # Define expected failures
+            EXPECTED_FAILS="src/tests/pysss-test.py3.sh pam-srv-tests ssh-srv-tests test_cert_utils sss_certmap_test sysdb-tests crypto-tests"
+
+            # Create temporary files for sorted lists
+            TEMP_DIR=$(mktemp -d)
+            ACTUAL_SORTED="${TEMP_DIR}/actual_sorted.txt"
+            EXPECTED_SORTED="${TEMP_DIR}/expected_sorted.txt"
+
+            # Clean and sort both lists
+            echo "$ACTUAL_FAILS" | tr ' ' '\n' | grep -v '^$' | sort > "$ACTUAL_SORTED"
+            echo "$EXPECTED_FAILS" | tr ' ' '\n' | grep -v '^$' | sort > "$EXPECTED_SORTED"
+
+            echo "DEBUG: Actual failed tests: $(tr '\n' ' ' < "$ACTUAL_SORTED")"
+            echo "DEBUG: Expected failed tests: $(tr '\n' ' ' < "$EXPECTED_SORTED")"
+
+            # Find missing in actual (in expected but not in actual)
+            MISSING=$(comm -23 "$EXPECTED_SORTED" "$ACTUAL_SORTED" | tr '\n' ' ')
+            # Find extra in actual (in actual but not in expected)
+            EXTRA=$(comm -13 "$EXPECTED_SORTED" "$ACTUAL_SORTED" | tr '\n' ' ')
+
+            # Clean up temporary files
+            rm -rf "$TEMP_DIR"
+
+            echo "Test(s) that should have failed: $MISSING"
+            echo "Test(s) that shouldn't have failed: $EXTRA"
+
+            if [ -z "$MISSING" ] && [ -z "$EXTRA" ]; then
+                echo "PASS: Actual failed tests match expected."
+                exit 0
+            else
+                echo "FAIL: Actual failed tests do not match expected."
+                exit 1
+            fi
+        else
+            echo "Error: sssd-test.log not found"
+            exit 1
+        fi
+    # ----- NGINX -----
+    elif [ "$TEST_SUITE" = "nginx" ]; then
+        if [ -f "nginx-test.log" ]; then
+            # Check if the test result shows FAIL
+            if grep -q "Result: FAIL" nginx-test.log; then
+                echo "PASS: nginx tests failed as expected with force fail enabled"
+                exit 0
+            else
+                echo "FAIL: nginx tests unexpectedly succeeded with force fail enabled"
+                exit 1
+            fi
+        else
+            echo "Error: nginx-test.log not found"
+            exit 1
+        fi
+    # ----- STUNNEL -----
+    elif [ "$TEST_SUITE" = "stunnel" ]; then
+        if [ -f "$GITHUB_WORKSPACE/tests/stunnel-test.log" ]; then
+            # Check for expected error patterns
+            if grep -q "failed: 41" "$GITHUB_WORKSPACE/tests/stunnel-test.log"; then
+                echo "PASS: stunnel tests failed as expected with force fail enabled"
+                exit 0
+            else
+                echo "FAIL: stunnel tests unexpectedly succeeded with force fail enabled"
+                exit 1
+            fi
+        else
+            echo "Error: stunnel-test.log not found"
+            exit 1
+        fi
     else
-        # --- generic force-fail logic for other suites ---
         if [ $TEST_RESULT -eq 0 ]; then
-            echo "Test unexpectedly succeeded with force fail enabled"
+            echo "$TEST_SUITE tests unexpectedly succeeded with force fail enabled"
             exit 1 # failure was not seen when expected
         else
-            echo "Test failed as expected with force fail enabled"
+            echo "$TEST_SUITE tests failed as expected with force fail enabled"
             exit 0 # expected failure occurred
         fi
     fi

.github/workflows/curl.yml

@@ -20,6 +20,7 @@ jobs:
     strategy:
       matrix:
         wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
+        openssl_ref: [ 'openssl-3.2.0' ]
     steps:
       - name: Checkout wolfProvider
         uses: actions/checkout@v4
@@ -49,7 +50,7 @@ jobs:
             openssl-source
             openssl-install
 
-          key: ossl-depends
+          key: ossl-depends-${{ matrix.openssl_ref }}
 
       # If not yet built this version, build it now
       - name: Build wolfProvider
@@ -73,7 +74,11 @@ jobs:
       matrix:
         curl_ref: [ 'master', 'curl-8_4_0' ]
         wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
+        openssl_ref: [ 'openssl-3.2.0' ]
         force_fail: ['WOLFPROV_FORCE_FAIL=1', '']
+        exclude:
+          - curl_ref: 'master'
+            force_fail: 'WOLFPROV_FORCE_FAIL=1'
     steps:
       - name: Checkout wolfProvider
         uses: actions/checkout@v4
@@ -86,7 +91,7 @@ jobs:
             openssl-source
             openssl-install
 
-          key: ossl-depends
+          key: ossl-depends-${{ matrix.openssl_ref }}
           fail-on-cache-miss: true
 
       - name: Retrieving wolfSSL/wolfProvider from cache

.github/workflows/grpc.yml

@@ -20,6 +20,7 @@ jobs:
     strategy:
       matrix:
         wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
+        openssl_ref: [ 'openssl-3.2.0' ]
     steps:
       - name: Checkout wolfProvider
         uses: actions/checkout@v4
@@ -49,7 +50,7 @@ jobs:
             openssl-source
             openssl-install
 
-          key: ossl-depends
+          key: ossl-depends-${{ matrix.openssl_ref }}
           fail-on-cache-miss: false
 
       # If not yet built this version, build it now
@@ -74,15 +75,22 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - ref: v1.60.0
+          - grpc_ref: [ 'master', 'v1.60.0' ]
             tests: >-
               bad_ssl_alpn_test bad_ssl_cert_test client_ssl_test
               crl_ssl_transport_security_test server_ssl_test
               ssl_transport_security_test ssl_transport_security_utils_test
               test_core_security_ssl_credentials_test test_cpp_end2end_ssl_credentials_test
               h2_ssl_cert_test h2_ssl_session_reuse_test
         wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
+        openssl_ref: [ 'openssl-3.2.0' ]
+        exclude:
+          - grpc_ref: 'master'
+            force_fail: 'WOLFPROV_FORCE_FAIL=1'
     steps:
+      - name: Checkout wolfProvider
+        uses: actions/checkout@v4
+
       - name: Confirm IPv4 and IPv6 support
         run: |
           ip addr list lo | grep 'inet '
@@ -96,7 +104,7 @@ jobs:
             openssl-source
             openssl-install
 
-          key: ossl-depends
+          key: ossl-depends-${{ matrix.openssl_ref }}
           fail-on-cache-miss: false
 
       - name: Retrieving wolfSSL/wolfProvider from cache
@@ -127,7 +135,7 @@ jobs:
         with:
           repository: grpc/grpc
           path: grpc
-          ref: ${{ matrix.ref }}
+          ref: ${{ matrix.grpc_ref }}
 
       - name: Build grpc with wolfProvider
         working-directory: ./grpc

.github/workflows/ipmitool.yml

@@ -20,6 +20,7 @@ jobs:
     strategy:
       matrix:
         wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
+        openssl_ref: [ 'openssl-3.2.0' ]
     steps:
       - name: Checkout wolfProvider
         uses: actions/checkout@v4
@@ -49,7 +50,7 @@ jobs:
             openssl-source
             openssl-install
 
-          key: ossl-depends
+          key: ossl-depends-${{ matrix.openssl_ref }}
           fail-on-cache-miss: false
 
       # If not yet built this version, build it now
@@ -73,8 +74,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        git_ref: [ c3939dac2c060651361fc71516806f9ab8c38901 ]
+        ipmitool_ref: [ 'master', 'c3939dac2c060651361fc71516806f9ab8c38901' ]
         wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
+        openssl_ref: [ 'openssl-3.2.0' ]
+        exclude:
+          - ipmitool_ref: 'master'
+            force_fail: 'WOLFPROV_FORCE_FAIL=1'
     steps:
       - name: Retrieving OpenSSL from cache
         uses: actions/cache/restore@v4
@@ -84,7 +89,7 @@ jobs:
             openssl-source
             openssl-install
 
-          key: ossl-depends
+          key: ossl-depends-${{ matrix.openssl_ref }}
           fail-on-cache-miss: false
 
       - name: Retrieving wolfSSL/wolfProvider from cache
@@ -110,9 +115,8 @@ jobs:
         uses: wolfSSL/actions-build-autotools-project@v1
         with:
           repository: ipmitool/ipmitool
-          ref: ${{ matrix.git_ref }}
+          ref: ${{ matrix.ipmitool_ref }}
           path: ipmitool
-          configure: --with-openssl=$GITHUB_WORKSPACE/openssl-install
           check: false
 
       - name: Confirm built with OpenSSL and test with wolfProvider

.github/workflows/multi-compiler.yml

@@ -111,4 +111,3 @@ jobs:
           if [ -f config.log ]; then
             cat config.log
           fi
-

.github/workflows/nginx.yml

@@ -20,6 +20,7 @@ jobs:
     strategy:
       matrix:
         wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
+        openssl_ref: [ 'openssl-3.2.0' ]
     steps:
       - name: Checkout wolfProvider
         uses: actions/checkout@v4
@@ -49,7 +50,7 @@ jobs:
             openssl-source
             openssl-install
 
-          key: ossl-depends
+          key: ossl-depends-${{ matrix.openssl_ref }}
 
       # If not yet built this version, build it now
       - name: Build wolfProvider
@@ -73,7 +74,15 @@ jobs:
       matrix:
         nginx_ref: [ 'master', 'release-1.27.4' ]
         wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
+        openssl_ref: [ 'openssl-3.2.0' ]
+        force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '']
+        exclude:
+          - nginx_ref: 'master'
+            force_fail: 'WOLFPROV_FORCE_FAIL=1'
     steps:
+      - name: Checkout wolfProvider
+        uses: actions/checkout@v4
+
       - name: Retrieving OpenSSL from cache
         uses: actions/cache/restore@v4
         id: openssl-cache
@@ -82,7 +91,7 @@ jobs:
             openssl-source
             openssl-install
 
-          key: ossl-depends
+          key: ossl-depends-${{ matrix.openssl_ref }}
           fail-on-cache-miss: true
 
       - name: Retrieving wolfSSL/wolfProvider from cache
@@ -125,7 +134,13 @@ jobs:
       - name: Run nginx-tests with wolfProvider
         working-directory: nginx-tests
         run: |
+          # Set environment variables
           export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/wolfssl-install/lib:$GITHUB_WORKSPACE/openssl-install/lib64
           export OPENSSL_CONF=$GITHUB_WORKSPACE/provider.conf
           export OPENSSL_MODULES=$GITHUB_WORKSPACE/wolfprov-install/lib
-          TEST_NGINX_VERBOSE=y TEST_NGINX_CATLOG=y TEST_NGINX_BINARY=../nginx/objs/nginx prove -v .
+          export ${{ matrix.force_fail }}
+
+          # Run tests and save result
+          TEST_NGINX_VERBOSE=y TEST_NGINX_CATLOG=y TEST_NGINX_BINARY=../nginx/objs/nginx prove -v . 2>&1 | tee nginx-test.log || true
+          TEST_RESULT=$?
+          $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} nginx