Skip to content

Commit 4ccfaa5

Browse files
padelsbachpadelsbach
committed
Unify workflow yamls with pre-processing step
1 parent 0c91a4d commit 4ccfaa5

File tree

13 files changed

+373
-38
lines changed

13 files changed

+373
-38
lines changed

‎.github/scripts/expand-workflow.sh‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
print_help() {
5+
cat <<'EOF'
6+
expand-includes.sh — simple textual include expander
7+
8+
USAGE:
9+
expand-includes.sh <input-file>
10+
expand-includes.sh -h | --help
11+
12+
DESCRIPTION:
13+
Reads <input-file> and writes the expanded result to stdout.
14+
15+
Lines matching the following form are replaced by the contents
16+
of the referenced file:
17+
18+
include: "path/to/file.yml.in" # optional comment
19+
20+
Rules:
21+
- The filepath MUST be in double quotes.
22+
- Leading whitespace (spaces and/or tabs) is preserved and applied
23+
to every line of the included file.
24+
- Trailing comments starting with '#' are ignored.
25+
- Whitespace around 'include:' and the filename is ignored.
26+
- Included files may themselves contain include: directives.
27+
- Expansion is purely textual; no YAML parsing is performed.
28+
29+
EOF
30+
}
31+
32+
expand_file() {
33+
local file="$1"
34+
35+
while IFS= read -r line || [[ -n "$line" ]]; do
36+
# Remove trailing comments for matching (keep original for output)
37+
local stripped="${line%%#*}"
38+
39+
# Match YAML include line:
40+
#
41+
# <indent>include: "path"
42+
#
43+
if [[ "$stripped" =~ ^([[:space:]]*)include[[:space:]]*:[[:space:]]*\"([^\"]+)\"[[:space:]]*$ ]]; then
44+
local indent="${BASH_REMATCH[1]}"
45+
local include_file="${BASH_REMATCH[2]}"
46+
47+
if [[ ! -f "$include_file" ]]; then
48+
echo "ERROR: included file not found: $include_file" >&2
49+
return 1
50+
fi
51+
52+
# Recursively expand included file (no subshells)
53+
if ! mapfile -t inc_lines < <(expand_file "$include_file"); then
54+
return 1
55+
fi
56+
57+
for inc_line in "${inc_lines[@]}"; do
58+
printf "%s%s\n" "$indent" "$inc_line"
59+
done
60+
61+
else
62+
echo "$line"
63+
fi
64+
done < "$file"
65+
}
66+
67+
68+
# ---- main ----
69+
70+
if [[ $# -ne 1 ]]; then
71+
print_help >&2
72+
exit 1
73+
fi
74+
75+
case "$1" in
76+
-h|--help)
77+
print_help
78+
exit 0
79+
;;
80+
esac
81+
82+
printf "# Warning: generated file, do not edit. Generated by '%s %s'\n" "$0" "$*"
83+
84+
expand_file "$1"

‎.github/workflow-src/Makefile‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
SHELL := /usr/bin/env bash
2+
3+
SRC_DIR := .
4+
OUT_DIR := ../workflows
5+
EXPAND := ../scripts/expand-workflow.sh
6+
7+
# All input files (recursively)
8+
INPUTS := $(shell find $(SRC_DIR) -type f)
9+
10+
# All top-level workflow inputs (one-to-one with outputs)
11+
WORKFLOW_SRCS := $(wildcard $(SRC_DIR)/*.yml.in)
12+
WORKFLOWS := $(patsubst $(SRC_DIR)/%.yml.in, $(OUT_DIR)/%.yml, $(WORKFLOW_SRCS))
13+
14+
.PHONY: all
15+
all: $(WORKFLOWS)
16+
17+
# Rule: output depends on *all* inputs and the expand script
18+
$(OUT_DIR)/%.yml: $(SRC_DIR)/%.yml.in $(INPUTS) $(EXPAND)
19+
@echo "Generating $@ from $<"
20+
@mkdir -p $(OUT_DIR)
21+
@$(EXPAND) $< > $@

‎.github/workflow-src/cjose.yml.in‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: cjose Tests
2+
include: "./common/wp-trigger.yml.in"
3+
4+
jobs:
5+
include: "./common/wp-build.yml.in"
6+
7+
test_cjose:
8+
include: "./common/wp-testenv.yml.in"
9+
10+
strategy:
11+
matrix:
12+
cjose_ref: [ 'v0.6.2.1' ]
13+
include: "./common/wp-matrix.yml.in"
14+
15+
steps:
16+
include: "./common/wp-install.yml.in"
17+
18+
- name: Install cjose dependencies
19+
run: |
20+
apt-get update
21+
apt-get install -y git build-essential autoconf automake \
22+
libtool pkg-config libjansson-dev check ca-certificates dpkg-dev
23+
24+
- name: Download cjose
25+
uses: actions/checkout@v4
26+
with:
27+
repository: OpenIDC/cjose
28+
ref: ${{ matrix.cjose_ref }}
29+
path: cjose
30+
fetch-depth: 1
31+
32+
- name: Build cjose
33+
working-directory: cjose
34+
run: |
35+
./configure CFLAGS="-Wno-error=deprecated-declarations"
36+
make
37+
38+
- name: Run cjose tests
39+
working-directory: cjose
40+
run: |
41+
export ${{ matrix.force_fail }}
42+
make test 2>&1 | tee cjose-test.log
43+
TEST_RESULT=$(grep -q "FAIL: check_cjose" cjose-test.log && echo "1" || echo "0")
44+
echo "TEST_RESULT = $TEST_RESULT"
45+
$GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} cjose

‎.github/workflow-src/common/wp-build.yml.in‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
build_wolfprovider:
2+
uses: ./.github/workflows/build-wolfprovider.yml
3+
with:
4+
wolfssl_ref: ${{ matrix.wolfssl_ref }}
5+
openssl_ref: ${{ matrix.openssl_ref }}
6+
fips_ref: ${{ matrix.fips_ref }}
7+
replace_default: ${{ matrix.replace_default }}
8+
strategy:
9+
matrix:
10+
include: "./common/wp-matrix.yml.in"

‎.github/workflow-src/common/wp-install.yml.in‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
- name: Setup env vars
2+
run: |
3+
# These paths must match the ones in build-wolfprovider.yml
4+
PACKAGES_PATH=/tmp
5+
echo "PACKAGES_PATH=$PACKAGES_PATH" >> "$GITHUB_ENV"
6+
echo "WOLFSSL_PACKAGES_PATH=\${PACKAGES_PATH}/wolfssl-packages" >> "$GITHUB_ENV"
7+
echo "OPENSSL_PACKAGES_PATH=\${PACKAGES_PATH}/openssl-packages" >> "$GITHUB_ENV"
8+
echo "WOLFPROV_PACKAGES_PATH=\${PACKAGES_PATH}/wolfprov-packages" >> "$GITHUB_ENV"
9+
10+
- name: Checkout wolfProvider
11+
uses: actions/checkout@v4
12+
with:
13+
fetch-depth: 1
14+
15+
- name: Download packages from build job
16+
uses: actions/download-artifact@v4
17+
with:
18+
name: debian-packages-${{ matrix.fips_ref }}${{ matrix.replace_default && '-replace-default' || '' }}-${{ matrix.wolfssl_ref }}-${{ matrix.openssl_ref }}
19+
path: ${{ env.PACKAGES_PATH }}
20+
21+
- name: Install wolfSSL/OpenSSL/wolfprov packages
22+
run: |
23+
apt install --reinstall -y \
24+
${{ env.WOLFSSL_PACKAGES_PATH }}/libwolfssl_*.deb
25+
26+
apt install --reinstall -y --allow-downgrades --allow-change-held-packages \
27+
${{ env.OPENSSL_PACKAGES_PATH }}/openssl_*.deb \
28+
${{ env.OPENSSL_PACKAGES_PATH }}/libssl3_*.deb \
29+
${{ env.OPENSSL_PACKAGES_PATH }}/libssl-dev_*.deb
30+
31+
apt install --reinstall -y \
32+
${{ env.WOLFPROV_PACKAGES_PATH }}/libwolfprov_*.deb
33+
34+
- name: Verify wolfProvider is properly installed
35+
run: |
36+
$GITHUB_WORKSPACE/scripts/verify-install.sh \
37+
${{ matrix.replace_default && '--replace-default' || '' }} \
38+
${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }}

‎.github/workflow-src/common/wp-matrix.yml.in‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
wolfssl_ref: [ 'v5.8.4-stable' ]
2+
openssl_ref: [ 'openssl-3.5.4' ]
3+
fips_ref: [ 'FIPS', 'non-FIPS' ]
4+
force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ]
5+
replace_default: [ true ]

‎.github/workflow-src/common/wp-testenv.yml.in‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
runs-on: ubuntu-22.04
2+
needs: build_wolfprovider
3+
# Run inside Debian Bookworm to match packaging environment
4+
container:
5+
image: debian:bookworm
6+
env:
7+
DEBIAN_FRONTEND: noninteractive
8+
9+
# This should be a safe limit for the tests to run.
10+
timeout-minutes: 20

‎.github/workflow-src/common/wp-trigger.yml.in‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
on:
2+
push:
3+
branches: [ 'master', 'main', 'release/**' ]
4+
pull_request:
5+
branches: [ '*' ]
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: true

‎.github/workflow-src/curl.yml.in‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: Curl Tests
2+
3+
include: "./common/wp-trigger.yml.in"
4+
5+
jobs:
6+
include: "./common/wp-build.yml.in"
7+
8+
test_curl:
9+
include: "./common/wp-testenv.yml.in"
10+
11+
strategy:
12+
matrix:
13+
curl_ref: [ 'curl-8_4_0', 'curl-7_88_1' ]
14+
include: "./common/wp-matrix.yml.in"
15+
steps:
16+
include: "./common/wp-install.yml.in"
17+
18+
- name: Install curl dependencies
19+
run: |
20+
apt-get update
21+
apt-get install -y nghttp2 libpsl5 libpsl-dev python3-impacket \
22+
build-essential autoconf automake libtool
23+
24+
- name: Build curl
25+
uses: wolfSSL/actions-build-autotools-project@v1
26+
with:
27+
repository: curl/curl
28+
path: curl
29+
ref: ${{ matrix.curl_ref }}
30+
configure: --with-openssl
31+
check: false
32+
33+
- name: Generate certificates for curl master force-fail tests
34+
run: |
35+
if [ "${{ matrix.force_fail }}" = "WOLFPROV_FORCE_FAIL=1" ] &&
36+
[ "${{ matrix.curl_ref }}" = "master" ]; then
37+
cd curl/tests/certs
38+
make test-ca.cacert
39+
cd ../..
40+
fi
41+
- name: Test curl with wolfProvider
42+
working-directory: curl
43+
shell: bash
44+
run: |
45+
set +o pipefail # ignore errors from make check
46+
export ${{ matrix.force_fail }}
47+
export CURL_REF=${{ matrix.curl_ref }}
48+
49+
# Tests rely on $USER being set
50+
export USER=testuser
51+
52+
# Run tests and save output to test.log
53+
make -j$(nproc) test-ci 2>&1 | tee curl-test.log
54+
# Capture the test result using PIPESTATUS (Bash only)
55+
TEST_RESULT=${PIPESTATUS[0]}
56+
$GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} curl

‎.github/workflow-src/readme.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Github workflow pre-processing
2+
3+
## TLDR
4+
Run `make` from this directory to regenerate the yamls in the `.github/workflows` directory. Then commit those files without further modification.
5+
6+
## Details
7+
In order to unify large chunks of code, we pre-process the workflow yamls to insert common blocks of code.
8+
9+
The files `*.yml.in` are the pre-processed workflow files. Manually edit these, not the generated ones in `.github/workflows`. Run `make` here to regenerate any outdated workflows.
10+
11+
The workflow `check-generated.yml` ensures that the pre-processing has been done.
12+
13+
### Syntax
14+
Designed to be as lightweight and yaml-like as possible, insert the following line structure into a `.yml.in` file.
15+
16+
` include: "path/to/file.yml.in"`
17+
18+
Any leading whitespace indentation is preserved.

0 commit comments

Comments
 (0)