Skip to content

Commit 3fa2227

Browse files
sylvestresylvestre
authored
Run against the GNU implementation (#223)
* Add a script to run the sed testsuite * run the gnu testsuite in the ci * remove useless comment
1 parent 2df1e00 commit 3fa2227

File tree

3 files changed

+819
-0
lines changed

3 files changed

+819
-0
lines changed

‎.github/workflows/GnuTests.yml‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
name: GnuTests
2+
3+
# Run GNU sed testsuite against the Rust sed implementation
4+
# This workflow extracts and runs tests from the GNU sed testsuite to ensure compatibility
5+
6+
on:
7+
pull_request:
8+
push:
9+
branches:
10+
- '*'
11+
12+
permissions:
13+
contents: read
14+
15+
# End the current execution if there is a new changeset in the PR
16+
concurrency:
17+
group: ${{ github.workflow }}-${{ github.ref }}
18+
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
19+
20+
env:
21+
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
22+
TEST_FULL_SUMMARY_FILE: 'sed-gnu-full-result.json'
23+
24+
jobs:
25+
native:
26+
name: Run GNU sed testsuite
27+
runs-on: ubuntu-24.04
28+
steps:
29+
#### Get the code, setup cache
30+
- name: Checkout code (sed)
31+
uses: actions/checkout@v4
32+
with:
33+
path: 'sed'
34+
persist-credentials: false
35+
- uses: dtolnay/rust-toolchain@master
36+
with:
37+
toolchain: stable
38+
components: rustfmt
39+
- uses: Swatinem/rust-cache@v2
40+
with:
41+
workspaces: "./sed -> target"
42+
- name: Checkout code (GNU sed testsuite)
43+
uses: actions/checkout@v4
44+
with:
45+
repository: 'mirror/sed'
46+
path: 'gnu.sed'
47+
ref: 'master'
48+
persist-credentials: false
49+
50+
#### Build environment setup
51+
- name: Install dependencies
52+
shell: bash
53+
run: |
54+
## Install dependencies for sed testing (no need to build GNU sed)
55+
sudo apt-get update
56+
sudo apt-get install -y jq perl
57+
58+
### Build
59+
- name: Build Rust sed binary
60+
shell: bash
61+
run: |
62+
## Build Rust sed binary
63+
cd 'sed'
64+
cargo build --release
65+
66+
### Run tests
67+
- name: Run GNU sed testsuite
68+
shell: bash
69+
run: |
70+
## Run GNU sed testsuite using our script
71+
cd 'sed'
72+
# Set GNU testsuite directory
73+
export GNU_TESTSUITE_DIR="../gnu.sed/testsuite"
74+
# Run tests with JSON output
75+
./util/run-gnu-testsuite.sh --json-output "${{ env.TEST_FULL_SUMMARY_FILE }}" || true
76+
77+
### Upload artifacts
78+
- name: Check for JSON results file
79+
shell: bash
80+
run: |
81+
echo "Checking for JSON results file..."
82+
ls -la sed/ || true
83+
ls -la sed/${{ env.TEST_FULL_SUMMARY_FILE }} || echo "JSON file not found at sed/${{ env.TEST_FULL_SUMMARY_FILE }}"
84+
ls -la ${{ env.TEST_FULL_SUMMARY_FILE }} || echo "JSON file not found at ${{ env.TEST_FULL_SUMMARY_FILE }}"
85+
find . -name "*json*" -type f || echo "No JSON files found"
86+
87+
- name: Upload full json results
88+
uses: actions/upload-artifact@v4
89+
with:
90+
name: sed-gnu-full-result
91+
path: sed/${{ env.TEST_FULL_SUMMARY_FILE }}
92+
if-no-files-found: warn
93+
94+
- name: Upload test logs
95+
if: always()
96+
uses: actions/upload-artifact@v4
97+
with:
98+
name: test-logs
99+
path: |
100+
sed/test-logs/*.log
101+
sed/test-results/*.json
102+
103+
aggregate:
104+
needs: [native]
105+
permissions:
106+
actions: read
107+
contents: read
108+
pull-requests: read
109+
name: Aggregate GNU test results
110+
runs-on: ubuntu-24.04
111+
steps:
112+
- name: Initialize workflow variables
113+
id: vars
114+
shell: bash
115+
run: |
116+
## VARs setup
117+
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
118+
119+
TEST_SUMMARY_FILE='sed-gnu-result.json'
120+
outputs TEST_SUMMARY_FILE
121+
122+
- name: Checkout code (sed)
123+
uses: actions/checkout@v4
124+
with:
125+
path: 'sed'
126+
persist-credentials: false
127+
128+
- name: Retrieve reference artifacts
129+
uses: dawidd6/action-download-artifact@v3
130+
continue-on-error: true
131+
with:
132+
workflow: GnuTests.yml
133+
branch: "${{ env.DEFAULT_BRANCH }}"
134+
workflow_conclusion: completed
135+
path: "reference"
136+
if_no_artifact_found: warn
137+
138+
- name: Download full json results
139+
uses: actions/download-artifact@v4
140+
with:
141+
name: sed-gnu-full-result
142+
path: results
143+
144+
- name: Extract/summarize testing info
145+
id: summary
146+
shell: bash
147+
run: |
148+
## Extract/summarize testing info
149+
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
150+
151+
# Check if results directory exists and has JSON files
152+
json_count=0
153+
if [[ -d "results" ]]; then
154+
json_count=$(find results -name "*.json" | wc -l)
155+
fi
156+
157+
if [[ "$json_count" -lt 1 ]]; then
158+
echo "::error ::Failed to download results json files; failing early"
159+
echo "::error ::Contents of results directory:"
160+
ls -lR results || echo "::error ::Results directory does not exist"
161+
exit 1
162+
fi
163+
164+
# Extract summary from JSON results
165+
RESULT_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}"
166+
if [[ -f "$RESULT_FILE" ]]; then
167+
TOTAL=$(jq -r '.summary.total // 0' "$RESULT_FILE")
168+
PASS=$(jq -r '.summary.passed // 0' "$RESULT_FILE")
169+
FAIL=$(jq -r '.summary.failed // 0' "$RESULT_FILE")
170+
SKIP=$(jq -r '.summary.skipped // 0' "$RESULT_FILE")
171+
ERROR=0 # Our format doesn't distinguish errors from failures
172+
else
173+
echo "::error ::Result file $RESULT_FILE not found"
174+
echo "::error ::Available files in results:"
175+
find results -type f || true
176+
TOTAL=0; PASS=0; FAIL=0; SKIP=0; ERROR=0
177+
fi
178+
179+
output="GNU sed tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP"
180+
echo "${output}"
181+
182+
if [[ "$FAIL" -gt 0 ]]; then
183+
echo "::warning ::${output}"
184+
fi
185+
186+
jq -n \
187+
--arg date "$(date --rfc-email)" \
188+
--arg sha "$GITHUB_SHA" \
189+
--arg total "$TOTAL" \
190+
--arg pass "$PASS" \
191+
--arg skip "$SKIP" \
192+
--arg fail "$FAIL" \
193+
--arg error "$ERROR" \
194+
'{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}'
195+
196+
HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1)
197+
outputs HASH TOTAL PASS FAIL SKIP
198+
199+
- name: Upload SHA1/ID of 'test-summary'
200+
uses: actions/upload-artifact@v4
201+
with:
202+
name: "${{ steps.summary.outputs.HASH }}"
203+
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
204+
205+
- name: Upload test results summary
206+
uses: actions/upload-artifact@v4
207+
with:
208+
name: test-summary
209+
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
210+
211+
- name: Report test results
212+
shell: bash
213+
run: |
214+
## Report final results
215+
echo "::notice ::GNU sed testsuite results:"
216+
echo "::notice :: Total tests: ${{ steps.summary.outputs.TOTAL }}"
217+
echo "::notice :: Passed: ${{ steps.summary.outputs.PASS }}"
218+
echo "::notice :: Failed: ${{ steps.summary.outputs.FAIL }}"
219+
echo "::notice :: Skipped: ${{ steps.summary.outputs.SKIP }}"
220+
221+
if [[ "${{ steps.summary.outputs.FAIL }}" -gt 0 ]]; then
222+
PASS_RATE=$(( ${{ steps.summary.outputs.PASS }} * 100 / (${{ steps.summary.outputs.PASS }} + ${{ steps.summary.outputs.FAIL }}) ))
223+
echo "::notice :: Pass rate: ${PASS_RATE}%"
224+
fi

‎README.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,31 @@ cargo run --release
4141

4242
The binary is named `sed` in `target/release/sed`.
4343

44+
## Testing
45+
46+
### GNU sed Compatibility Testing
47+
48+
Test compatibility against GNU sed using the comprehensive testsuite (47+ tests, ~10% pass rate):
49+
50+
```bash
51+
# Clone GNU sed testsuite (one time setup)
52+
git clone https://github.com/mirror/sed.git ../gnu.sed
53+
54+
# Run compatibility tests
55+
./util/run-gnu-testsuite.sh
56+
57+
# Generate JSON results for CI
58+
./util/run-gnu-testsuite.sh --json-output results.json
59+
```
60+
61+
The testsuite extracts test cases from the GNU sed repository and tests them against expected outputs.
62+
63+
### Unit Tests
64+
65+
```bash
66+
cargo test
67+
```
68+
4469
## Extensions and incompatibilities
4570
### Supported GNU extensions
4671
* Command-line arguments can be specified in long (`--`) form.

0 commit comments

Comments
 (0)