Skip to content

Commit 379123d

Browse files
authored
Improvements to CI code coverage (#796)
- Add maint/RunCoverage script to allow gathering Clang coverage locally more easily - Add LCOV exclusion comments to unreachable code - Add a filtering script so that the CI job respects LCOV_EXCL_* comments
1 parent b0f27ec commit 379123d

20 files changed

+519
-77
lines changed

.github/workflows/build.yml

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ jobs:
531531
- name: Setup
532532
run: |
533533
sudo apt-get -qq update
534-
sudo apt-get -qq install zlib1g-dev libbz2-dev libedit-dev
534+
sudo apt-get -qq install zlib1g-dev libbz2-dev libedit-dev lcov
535535
536536
- name: Checkout
537537
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -547,50 +547,13 @@ jobs:
547547
- name: Test
548548
run: |
549549
cd build
550-
echo "== Running all tests with CTest =="
551-
LLVM_PROFILE_FILE="coverage-%m.profraw" ctest -j1 --output-on-failure
552-
echo ""
553-
echo "== Re-running pcre2test with -malloc =="
554-
LLVM_PROFILE_FILE="coverage-%m.profraw" srcdir=.. pcre2test=./pcre2test ../RunTest -malloc
555-
556-
- name: Report
557-
run: |
558-
LLVM_VER=`clang --version | head -n1 | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f1`
559-
echo "Using LLVM version $LLVM_VER"
560-
561-
# Merge the profiles gathered
562-
cd build
563-
llvm-profdata-$LLVM_VER merge -sparse coverage-*.profraw -o coverage.profdata
564-
565-
# Output HTML, for archiving and browsing later
566-
llvm-cov-$LLVM_VER show \
567-
-format=html -output-dir=coverage-report -show-line-counts-or-regions -show-branches=percent \
568-
-instr-profile=coverage.profdata \
569-
./pcre2test -object ./pcre2grep -object ./pcre2posix_test -object ./pcre2_jit_test \
570-
../src/ ./
571-
572-
# Output LCOV-compatible output, for downstream tools
573-
llvm-cov-$LLVM_VER export \
574-
-format=lcov \
575-
-instr-profile=coverage.profdata \
576-
./pcre2test -object ./pcre2grep -object ./pcre2posix_test -object ./pcre2_jit_test \
577-
../src/ ./ \
578-
> ./coverage-lcov.info
579-
580-
# Output text summary to build log
581-
echo '```' > "$GITHUB_STEP_SUMMARY"
582-
llvm-cov-$LLVM_VER report \
583-
-instr-profile=coverage.profdata \
584-
./pcre2test -object ./pcre2grep -object ./pcre2posix_test -object ./pcre2_jit_test \
585-
../src/ ./ \
586-
>> "$GITHUB_STEP_SUMMARY"
587-
echo '```' >> "$GITHUB_STEP_SUMMARY"
550+
../maint/RunCoverage
588551
589552
- name: Upload report to GitHub artifacts
590553
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
591554
with:
592555
name: "Coverage report"
593-
path: './build/coverage-report'
556+
path: './build/coverage-html'
594557
if-no-files-found: error
595558

596559
- name: Upload report to Codecov

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ build-*/
66
*.a
77
*.gcda
88
*.gcno
9+
*.profraw
910
*.lo
1011
*.la
1112
*.pc

RunGrepTest

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1148,6 +1148,9 @@ checkspecial '--line-buffered --colour=auto abc /dev/null' 1
11481148
checkspecial '--line-buffered --color abc /dev/null' 1
11491149
checkspecial '-dskip abc .' 1
11501150
checkspecial '-Dread -Dskip abc /dev/null' 1
1151+
checkspecial "-f $srcdir/testdata/greplistBad /dev/null" 2
1152+
checkspecial "(unpaired /dev/null" 2
1153+
checkspecial "-e (unpaired1 -e (unpaired2 /dev/null" 2
11511154

11521155
# Clean up local working files
11531156
rm -f testNinputgrep teststderrgrep testtrygrep testtemp1grep testtemp2grep

RunGrepTest.bat

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1089,6 +1089,9 @@ call :checkspecial "--line-buffered --colour=auto abc nul" 1 || exit /b 1
10891089
call :checkspecial "--line-buffered --color abc nul" 1 || exit /b 1
10901090
call :checkspecial "-dskip abc ." 1 || exit /b 1
10911091
call :checkspecial "-Dread -Dskip abc nul" 1 || exit /b 1
1092+
call :checkspecial "-f %srcdir%\testdata\greplistBad nul" 2 || exit /b 1
1093+
call :checkspecial "(unpaired nul" 2 || exit /b 1
1094+
call :checkspecial "-e (unpaired1 -e (unpaired2 nul" 2 || exit /b 1
10921095

10931096

10941097
:: Clean up local working files

RunTest

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -540,9 +540,14 @@ for bmode in "$test8" "$test16" "$test32"; do
540540
saverc=0
541541
checkspecial '-C' || saverc=$?
542542
checkspecial '--help' || saverc=$?
543+
checkspecial "$bmode testSinput" || saverc=$?
544+
checkspecial "$bmode $testdata/testinputheap" || saverc=$?
543545
if [ $support_setstack -eq 0 ] ; then
544-
checkspecial '-S 1 -t 10 testSinput' || saverc=$?
546+
checkspecial "$bmode -S 1 -t 10 testSinput" || saverc=$?
545547
fi
548+
checkspecial -LM || saverc=$?
549+
checkspecial -LP || saverc=$?
550+
checkspecial -LS || saverc=$?
546551
if [ $saverc -eq 0 ] ; then
547552
echo " OK"
548553
fi

maint/FilterCoverage.py

Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,271 @@
1+
#! /usr/bin/env python3
2+
3+
# Script which is a simple LCOV filter: removes DA/BRDA entries for lines marked in-source with
4+
# "LCOV_EXCL_LINE" or "LCOV_EXCL_START"/"LCOV_EXCL_STOP".
5+
#
6+
# Usage: python3 FilterCoverage.py coverage-lcov.info > coverage-lcov.filtered.info
7+
8+
import sys
9+
import re
10+
11+
def scan_exclusions(srcpath):
12+
"""Return a set of line numbers to exclude for this source file."""
13+
with open(srcpath, "r", encoding="utf-8") as fh:
14+
text = fh.readlines()
15+
excl = set()
16+
in_block = False
17+
for i, line in enumerate(text, start=1):
18+
if "LCOV_EXCL_LINE" in line:
19+
excl.add(i)
20+
if "LCOV_EXCL_START" in line:
21+
in_block = True
22+
excl.add(i)
23+
continue
24+
if "LCOV_EXCL_STOP" in line:
25+
excl.add(i)
26+
in_block = False
27+
continue
28+
if in_block:
29+
excl.add(i)
30+
return excl
31+
32+
DA_RE = re.compile(r'^\s*DA:(\d+),(\d+)(,.*)?\s*$')
33+
LF_RE = re.compile(r'^\s*LF:(\d+)\s*$')
34+
LH_RE = re.compile(r'^\s*LH:(\d+)\s*$')
35+
BRDA_RE = re.compile(r'^\s*BRDA:(\d+),([e\d]+),(.*),([-\d]+)\s*$')
36+
BRF_RE = re.compile(r'^\s*BRF:(\d+)\s*$')
37+
BRH_RE = re.compile(r'^\s*BRH:(\d+)\s*$')
38+
FN_RE = re.compile(r'^\s*FN:(\d+),([^,\s]*)\s*$')
39+
FNDA_RE = re.compile(r'^\s*FNDA:(\d+),([^,\s]*)\s*$')
40+
FNF_RE = re.compile(r'^\s*FNF:(\d+)\s*$')
41+
FNH_RE = re.compile(r'^\s*FNH:(\d+)\s*$')
42+
43+
def process_block(block_lines):
44+
"""Return processed block lines with excluded DA/BRDA removed and LF/LH fixed."""
45+
if not block_lines:
46+
return block_lines
47+
# get SF path from first 'SF:' line (should be first)
48+
first = block_lines[0]
49+
assert first.lstrip().startswith('SF:')
50+
sf_path = first.split(':', 1)[1].strip()
51+
exclusions = scan_exclusions(sf_path)
52+
53+
new_lines = []
54+
da_orig_found = 0
55+
da_orig_hit = 0
56+
da_new_found = 0
57+
da_new_hit = 0
58+
brda_orig_found = 0
59+
brda_orig_hit = 0
60+
brda_new_found = 0
61+
brda_new_hit = 0
62+
fnda_orig_found = 0
63+
fnda_orig_hit = 0
64+
fnda_new_found = 0
65+
fnda_new_hit = 0
66+
67+
fn_exclusions = set()
68+
69+
# Pass 1: identify FN exclusions
70+
for line in block_lines:
71+
m_fn = FN_RE.match(line)
72+
assert (m_fn is not None) == line.lstrip().startswith('FN:')
73+
if m_fn:
74+
fn_line = int(m_fn.group(1))
75+
fn_name = m_fn.group(2)
76+
if fn_line in exclusions:
77+
fn_exclusions.add(fn_name)
78+
79+
# Pass 2: filter DA, BRDA, FN/FNDA; copy others verbatim
80+
for line in block_lines:
81+
m_da = DA_RE.match(line)
82+
assert (m_da is not None) == line.lstrip().startswith('DA:')
83+
if m_da:
84+
line_num = int(m_da.group(1))
85+
execution_count = int(m_da.group(2))
86+
da_orig_found += 1
87+
if execution_count > 0:
88+
da_orig_hit += 1
89+
if line_num in exclusions:
90+
# drop this DA line
91+
continue
92+
da_new_found += 1
93+
if execution_count > 0:
94+
da_new_hit += 1
95+
new_lines.append(line)
96+
continue
97+
m_brda = BRDA_RE.match(line)
98+
assert (m_brda is not None) == line.lstrip().startswith('BRDA:')
99+
if m_brda:
100+
brda_orig_found += 1
101+
taken = m_brda.group(4)
102+
if taken != '-' and int(taken) > 0:
103+
brda_orig_hit += 1
104+
if int(m_brda.group(1)) in exclusions:
105+
# drop this BRDA line
106+
continue
107+
brda_new_found += 1
108+
if taken != '-' and int(taken) > 0:
109+
brda_new_hit += 1
110+
new_lines.append(line)
111+
continue
112+
m_fnda = FNDA_RE.match(line)
113+
assert (m_fnda is not None) == line.lstrip().startswith('FNDA:')
114+
if m_fnda:
115+
fnda_orig_found += 1
116+
fn_name = m_fnda.group(2)
117+
count = int(m_fnda.group(1))
118+
if count > 0:
119+
fnda_orig_hit += 1
120+
if fn_name in fn_exclusions:
121+
# drop this FNDA line
122+
continue
123+
fnda_new_found += 1
124+
if count > 0:
125+
fnda_new_hit += 1
126+
new_lines.append(line)
127+
continue
128+
m_fn = FN_RE.match(line)
129+
assert (m_fn is not None) == line.lstrip().startswith('FN:')
130+
if m_fn:
131+
fn_line = int(m_fn.group(1))
132+
fn_name = m_fn.group(2)
133+
if fn_name in fn_exclusions:
134+
# drop this FN line
135+
continue
136+
new_lines.append(line)
137+
continue
138+
# other lines: append unchanged
139+
new_lines.append(line)
140+
141+
# Pass 3: fix LF/LH, BRF/BRH, FNF/FNH
142+
# Mutate new_lines. If we find any LF/LH lines, check they have the expected original values.
143+
# If so, replace with new values. If not, print a warning.
144+
for i, line in enumerate(new_lines):
145+
# LF
146+
m_lf = LF_RE.match(line)
147+
assert (m_lf is not None) == line.lstrip().startswith('LF:')
148+
if m_lf:
149+
# preserve leading whitespace exactly
150+
leading = re.match(r'^(\s*)', line).group(1)
151+
# replace with recomputed value (number of DA entries remaining)
152+
new_lines[i] = f"{leading}LF:{da_new_found}\n"
153+
# warn if original disagreed (useful for debugging)
154+
try:
155+
lf_orig = int(m_lf.group(1))
156+
if lf_orig != da_orig_found:
157+
print(f"warning: original LF ({lf_orig}) != counted DA entries ({da_orig_found}) for {sf_path}", file=sys.stderr)
158+
except Exception:
159+
pass
160+
continue
161+
162+
# LH
163+
m_lh = LH_RE.match(line)
164+
assert (m_lh is not None) == line.lstrip().startswith('LH:')
165+
if m_lh:
166+
leading = re.match(r'^(\s*)', line).group(1)
167+
new_lines[i] = f"{leading}LH:{da_new_hit}\n"
168+
try:
169+
lh_orig = int(m_lh.group(1))
170+
if lh_orig != da_orig_hit:
171+
print(f"warning: original LH ({lh_orig}) != counted DA hits ({da_orig_hit}) for {sf_path}", file=sys.stderr)
172+
except Exception:
173+
pass
174+
continue
175+
176+
# BRF
177+
m_brf = BRF_RE.match(line)
178+
assert (m_brf is not None) == line.lstrip().startswith('BRF:')
179+
if m_brf:
180+
leading = re.match(r'^(\s*)', line).group(1)
181+
# replace with recomputed branch-found (if you computed brda_new_found above)
182+
new_lines[i] = f"{leading}BRF:{brda_new_found}\n"
183+
try:
184+
brf_orig = int(m_brf.group(1))
185+
if brf_orig != brda_orig_found:
186+
print(f"warning: original BRF ({brf_orig}) != counted BRDA entries ({brda_orig_found}) for {sf_path}", file=sys.stderr)
187+
except Exception:
188+
pass
189+
continue
190+
191+
# BRH
192+
m_brh = BRH_RE.match(line)
193+
assert (m_brh is not None) == line.lstrip().startswith('BRH:')
194+
if m_brh:
195+
leading = re.match(r'^(\s*)', line).group(1)
196+
new_lines[i] = f"{leading}BRH:{brda_new_hit}\n"
197+
try:
198+
brh_orig = int(m_brh.group(1))
199+
if brh_orig != brda_orig_hit:
200+
print(f"warning: original BRH ({brh_orig}) != counted BRDA hits ({brda_orig_hit}) for {sf_path}", file=sys.stderr)
201+
except Exception:
202+
pass
203+
continue
204+
205+
# FNF
206+
m_fnf = FNF_RE.match(line)
207+
assert (m_fnf is not None) == line.lstrip().startswith('FNF:')
208+
if m_fnf:
209+
leading = re.match(r'^(\s*)', line).group(1)
210+
new_lines[i] = f"{leading}FNF:{fnda_new_found}\n"
211+
try:
212+
fnf_orig = int(m_fnf.group(1))
213+
if fnf_orig != fnda_orig_found:
214+
print(f"warning: original FNF ({fnf_orig}) != counted FNDA entries ({fnda_orig_found}) for {sf_path}", file=sys.stderr)
215+
except Exception:
216+
pass
217+
continue
218+
219+
# FNH
220+
m_fnh = FNH_RE.match(line)
221+
assert (m_fnh is not None) == line.lstrip().startswith('FNH:')
222+
if m_fnh:
223+
leading = re.match(r'^(\s*)', line).group(1)
224+
new_lines[i] = f"{leading}FNH:{fnda_new_hit}\n"
225+
try:
226+
fnh_orig = int(m_fnh.group(1))
227+
if fnh_orig != fnda_orig_hit:
228+
print(f"warning: original FNH ({fnh_orig}) != counted FNDA hits ({fnda_orig_hit}) for {sf_path}", file=sys.stderr)
229+
except Exception:
230+
pass
231+
continue
232+
233+
return new_lines
234+
235+
def filter_lcov(in_fh, out_fh):
236+
lines = in_fh.readlines()
237+
238+
i = 0
239+
out_lines = []
240+
while i < len(lines):
241+
line = lines[i]
242+
if line.lstrip().startswith('SF:'):
243+
# buffer block until end_of_record
244+
block = []
245+
while i < len(lines):
246+
block.append(lines[i])
247+
if lines[i].strip() == 'end_of_record':
248+
i += 1
249+
break
250+
i += 1
251+
processed = process_block(block)
252+
out_lines.extend(processed)
253+
else:
254+
out_lines.append(line)
255+
i += 1
256+
257+
out_fh.writelines(out_lines)
258+
259+
if __name__ == "__main__":
260+
if len(sys.argv) > 3:
261+
print("Usage: python3 FilterCoverage.py [infile [outfile]]", file=sys.stderr)
262+
sys.exit(1)
263+
if len(sys.argv) > 2:
264+
with open(sys.argv[2], "w", encoding="utf-8") as out_fh:
265+
with open(sys.argv[1], "r", encoding="utf-8") as in_fh:
266+
filter_lcov(in_fh, out_fh)
267+
elif len(sys.argv) > 1:
268+
with open(sys.argv[1], "r", encoding="utf-8") as fh:
269+
filter_lcov(fh, sys.stdout)
270+
else:
271+
filter_lcov(sys.stdin, sys.stdout)

maint/README

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ GenerateUcpTables.py
6060
GenerateCommon.py and Unicode data files. The generated file contains tables
6161
for looking up Unicode property names.
6262

63+
FilterCoverage.py
64+
A small helper used by the RunCoverage script.
65+
6366
LintMan
6467
A Perl script to check and update magic numbers in the documentation that
6568
correspond to configurable settings in the codebase.
@@ -95,6 +98,10 @@ pcre2_chartables.c.non-standard
9598
README
9699
This file.
97100

101+
RunCoverage
102+
A script used to generate the coverage report using Clang. It is called by
103+
the GitHub CI actions, and can also be run by a developer locally.
104+
98105
RunManifestTest
99106
RunManifestTest.ps1
100107
Scripts to generate and verify a list of files against an expected 'manifest'

0 commit comments

Comments
 (0)