Skip to content

Commit be19a6b

Browse files
ColtonWilleyColtonWilley
authored
Merge pull request #119 from aidangarske/curl-WPFF-workflow
Add `WOLFPROV_FORCE_FAIL=1` option to curl.yml test
2 parents d58a1b8 + bfc28bb commit be19a6b

File tree

3 files changed

+170
-35
lines changed

3 files changed

+170
-35
lines changed

‎.github/scripts/check-workflow-result.sh‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
if [ $# -lt 2 ]; then
6+
echo "Usage: $0 <test_result> [WOLFPROV_FORCE_FAIL] [TEST_SUITE]"
7+
exit 1
8+
fi
9+
10+
TEST_RESULT="$1"
11+
WOLFPROV_FORCE_FAIL="${2:-}"
12+
TEST_SUITE="${3:-}"
13+
14+
if [ "$WOLFPROV_FORCE_FAIL" = "1" ]; then
15+
if [ "$TEST_SUITE" = "curl" ]; then
16+
# --- curl-specific logic ---
17+
if [ -f "tests/test.log" ]; then
18+
# Extract and clean the failed test list from the log
19+
ACTUAL_FAILS=$(grep -a '^TESTFAIL: These test cases failed:' tests/test.log | sed 's/.*failed: //')
20+
else
21+
echo "Error: tests/test.log not found"
22+
exit 1
23+
fi
24+
25+
# Get curl version from the workflow ref
26+
CURL_VERSION="${CURL_REF:-}"
27+
28+
# Define expected failures based on curl version
29+
case "$CURL_VERSION" in
30+
"curl-8_4_0")
31+
EXPECTED_FAILS="9 31 39 41 44 46 61 64 65 70 71 72 73 88 153 154 158 163 166 167 168 169 170 171 173 186 206 245 246 258 259 273 277 327 335 388 420 444 540 551 552 554 565 579 584 643 645 646 647 648 649 650 651 652 653 654 666 667 668 669 670 671 672 673 977 1001 1002 1030 1053 1060 1061 1071 1072 1079 1095 1105 1133 1136 1151 1155 1158 1160 1161 1186 1187 1189 1190 1191 1192 1193 1194 1195 1196 1198 1199 1229 1284 1285 1286 1293 1315 1404 1412 1415 1418 1437 1568 1903 1905 1916 1917 1964 2024 2026 2027 2028 2030 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2073 2076 2200 2201 2202 2203 2204 3017 3018"
32+
;;
33+
"master")
34+
EXPECTED_FAILS="9 31 39 41 44 46 61 64 65 70 71 72 73 88 153 154 158 163 166 167 168 169 170 171 173 186 206 245 246 258 259 273 277 327 335 388 420 444 483 540 551 552 554 565 579 584 643 645 646 647 648 649 650 651 652 653 654 666 667 668 669 670 671 672 673 695 977 1001 1002 1030 1053 1060 1061 1071 1072 1079 1095 1105 1133 1136 1151 1155 1158 1160 1161 1186 1187 1189 1190 1191 1192 1193 1194 1195 1196 1198 1199 1229 1284 1285 1286 1293 1315 1404 1412 1415 1418 1437 1476 1568 1608 1610 1615 1654 1660 1903 1905 1916 1917 1964 2024 2026 2027 2028 2030 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2073 2076 2200 2201 2202 2203 2204 3017 3018"
35+
;;
36+
*)
37+
echo "Error: Unknown curl version: $CURL_VERSION"
38+
exit 1
39+
;;
40+
esac
41+
42+
# Create temporary files for sorted lists
43+
TEMP_DIR=$(mktemp -d)
44+
ACTUAL_SORTED="${TEMP_DIR}/actual_sorted.txt"
45+
EXPECTED_SORTED="${TEMP_DIR}/expected_sorted.txt"
46+
47+
# Clean and sort both lists and remove empty lines
48+
echo "$ACTUAL_FAILS" | tr ' ' '\n' | grep -v '^$' | sort -n > "$ACTUAL_SORTED"
49+
echo "$EXPECTED_FAILS" | tr ' ' '\n' | grep -v '^$' | sort -n > "$EXPECTED_SORTED"
50+
51+
echo "DEBUG: Sorted actual fails: $(tr '\n' ' ' < "$ACTUAL_SORTED")"
52+
echo "DEBUG: Sorted expected fails: $(tr '\n' ' ' < "$EXPECTED_SORTED")"
53+
54+
# Find missing in actual (in expected but not in actual)
55+
MISSING=$(comm -23 "$EXPECTED_SORTED" "$ACTUAL_SORTED" | tr '\n' ' ')
56+
# Find extra in actual (in actual but not in expected)
57+
EXTRA=$(comm -13 "$EXPECTED_SORTED" "$ACTUAL_SORTED" | tr '\n' ' ')
58+
59+
# Clean up temporary files
60+
rm -rf "$TEMP_DIR"
61+
62+
echo "Test(s) that should have failed: $MISSING"
63+
echo "Test(s) that shouldn't have failed: $EXTRA"
64+
65+
if [ -z "$MISSING" ] && [ -z "$EXTRA" ]; then
66+
echo "PASS: Actual failed tests match expected."
67+
exit 0
68+
else
69+
echo "FAIL: Actual failed tests do not match expected."
70+
exit 1
71+
fi
72+
elif [ "$TEST_SUITE" = "simple" ]; then
73+
# --- simple test suite specific logic ---
74+
if [ -f "test-suite.log" ]; then
75+
# For simple tests, we expect all tests to fail when force fail is enabled
76+
if [ $TEST_RESULT -eq 0 ]; then
77+
echo "Simple tests unexpectedly succeeded with force fail enabled"
78+
exit 1
79+
else
80+
echo "Simple tests failed as expected with force fail enabled"
81+
exit 0
82+
fi
83+
else
84+
echo "Error: test-suite.log not found"
85+
exit 1
86+
fi
87+
else
88+
# --- generic force-fail logic for other suites ---
89+
if [ $TEST_RESULT -eq 0 ]; then
90+
echo "Test unexpectedly succeeded with force fail enabled"
91+
exit 1 # failure was not seen when expected
92+
else
93+
echo "Test failed as expected with force fail enabled"
94+
exit 0 # expected failure occurred
95+
fi
96+
fi
97+
elif [ $TEST_RESULT -ne 0 ]; then
98+
echo "Tests failed unexpectedly"
99+
exit 1
100+
else
101+
echo "Tests passed successfully"
102+
exit 0
103+
fi

‎.github/workflows/curl.yml‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,11 @@ jobs:
7373
matrix:
7474
curl_ref: [ 'master', 'curl-8_4_0' ]
7575
wolfssl_ref: [ 'master', 'v5.7.4-stable' ]
76+
force_fail: [ 1, 0 ] # ['WOLFPROV_FORCE_FAIL=1', '']
7677
steps:
78+
- name: Checkout wolfProvider
79+
uses: actions/checkout@v4
80+
7781
- name: Retrieving OpenSSL from cache
7882
uses: actions/cache/restore@v4
7983
id: openssl-cache
@@ -109,13 +113,30 @@ jobs:
109113
repository: curl/curl
110114
path: curl
111115
ref: ${{ matrix.curl_ref }}
112-
configure: --with-openssl=$GITHUB_WORKSPACE/openssl-install/
116+
configure: --with-openssl
113117
check: false
114118

119+
- name: Generate certificates for curl master force-fail tests
120+
run: |
121+
if [ "${{ matrix.force_fail }}" = "1" ] &&
122+
[ "${{ matrix.curl_ref }}" = "master" ]; then
123+
cd curl/tests/certs
124+
make test-ca.cacert
125+
cd ../..
126+
fi
115127
- name: Test curl with wolfProvider
116128
working-directory: curl
117129
run: |
130+
# Set environment variables
118131
export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/wolfssl-install/lib:$GITHUB_WORKSPACE/openssl-install/lib64
119132
export OPENSSL_CONF=$GITHUB_WORKSPACE/provider.conf
120133
export OPENSSL_MODULES=$GITHUB_WORKSPACE/wolfprov-install/lib
121-
make -j $(nproc) test-ci
134+
export PKG_CONFIG_PATH=$GITHUB_WORKSPACE/openssl-install/lib64/pkgconfig
135+
export WOLFPROV_FORCE_FAIL=${{ matrix.force_fail }}
136+
export CURL_REF=${{ matrix.curl_ref }}
137+
138+
# Run tests and save output to test.log
139+
mkdir -p tests
140+
make -j$(nproc) test-ci 2>&1 | tee tests/test.log || true
141+
TEST_RESULT=$?
142+
$GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} curl

‎.github/workflows/simple.yml‎

Lines changed: 44 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ name: Simple Tests
33
# START OF COMMON SECTION
44
on:
55
push:
6-
branches: [ '*' ]
7-
# branches: [ 'master', 'main', 'release/**' ]
6+
branches: [ 'master', 'main', 'release/**' ]
87
pull_request:
98
branches: [ '*' ]
109

@@ -14,53 +13,65 @@ concurrency:
1413
# END OF COMMON SECTION
1514

1615
jobs:
17-
make_check:
18-
runs-on: ubuntu-latest
16+
simple_test:
17+
name: Simple Test
18+
runs-on: ubuntu-22.04
1919
timeout-minutes: 20
20-
2120
strategy:
2221
matrix:
23-
config:
22+
build_ref:
2423
- ''
2524
- 'OPENSSL_TAG=master'
2625
- 'WOLFSSL_TAG=master'
2726
- 'OPENSSL_TAG=master WOLFSSL_TAG=master'
28-
force_fail:
29-
- ''
30-
- 'WOLFPROV_FORCE_FAIL=1'
27+
force_fail: [ 1, 0 ] # ['WOLFPROV_FORCE_FAIL=1', '']
3128

3229
steps:
33-
- uses: actions/checkout@v4
34-
name: Checkout repository
30+
- name: Checkout wolfProvider
31+
uses: actions/checkout@v4
3532

36-
- name: Run build and tests
37-
run: |
38-
# Build first with matrix config
39-
${{ matrix.config }} ${{ matrix.force_fail }} ./scripts/build-wolfprovider.sh || BUILD_RESULT=$?
33+
# Check if this version of wolfssl/wolfprovider has already been built,
34+
# mark to cache these items on post if we do end up building
35+
- name: Checking wolfSSL/wolfProvider in cache
36+
uses: actions/cache@v4
37+
id: wolfprov-cache
38+
with:
39+
path: |
40+
wolfssl-source
41+
wolfssl-install
42+
wolfprov-install
43+
provider.conf
4044
41-
# Run all tests regardless of build result
42-
${{ matrix.force_fail }} ./scripts/cmd_test/do-cmd-tests.sh || TEST_RESULT=$?
45+
key: wolfprov-${{ matrix.build_ref }}-${{ github.sha }}
46+
lookup-only: true
4347

44-
# For force_fail, we expect failures (return 1)
45-
if [ -n "${{ matrix.force_fail }}" ]; then
46-
if [ $BUILD_RESULT -eq 0 ] || [ $TEST_RESULT -eq 0 ]; then
47-
echo "Build/Test unexpectedly succeeded with force fail enabled"
48-
exit 1 # failure was not seen when expected
49-
else
50-
echo "Build/Test failed as expected with force fail enabled"
51-
exit 0 # expected failure occurred
52-
fi
53-
else
54-
# Normal case - expect success
55-
if [ $BUILD_RESULT -ne 0 ] || [ $TEST_RESULT -ne 0 ]; then
56-
exit 1 # unexpected failure
57-
fi
58-
fi
48+
# If wolfssl/wolfprovider have not yet been built, pull ossl from cache
49+
- name: Checking OpenSSL in cache
50+
if: steps.wolfprov-${{ matrix.build_ref }}-cache.hit != 'true'
51+
uses: actions/cache@v4
52+
id: openssl-cache
53+
with:
54+
path: |
55+
openssl-source
56+
openssl-install
57+
58+
key: ossl-depends
59+
60+
# If not yet built this version, build it now
61+
- name: Build wolfProvider
62+
if: steps.wolfprov-${{ matrix.build_ref }}-cache.hit != 'true'
63+
run: |
64+
${{ matrix.build_ref.openssl }} ${{ matrix.build_ref.wolfssl }} WOLFPROV_FORCE_FAIL=${{ matrix.force_fail }} ./scripts/build-wolfprovider.sh || BUILD_RESULT=$?
65+
$GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $BUILD_RESULT ${{ matrix.force_fail }} simple
66+
67+
- name: Run simple tests
68+
run: |
69+
WOLFPROV_FORCE_FAIL=${{ matrix.force_fail }} ./scripts/cmd_test/do-cmd-tests.sh || TEST_RESULT=$?
70+
$GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} simple
5971
6072
- name: Print test logs
6173
if: always()
6274
run: |
6375
if [ -f test-suite.log ] ; then
6476
cat test-suite.log
6577
fi
66-

0 commit comments

Comments
 (0)