Skip to content

Commit 92698bb

Browse files
commjoencommjoen
authored
Merge pull request #2120 from OWASP/copilot/fix-424
Add comprehensive secret scanner comparison GitHub Action workflow
2 parents 7a82182 + cc3af13 commit 92698bb

File tree

4 files changed

+409
-1
lines changed

4 files changed

+409
-1
lines changed

‎.github/workflows/scanner-comparison.yml‎

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
---
2+
name: Secret Scanner Comparison Benchmark
3+
4+
on:
5+
workflow_dispatch:
6+
schedule:
7+
# Run weekly on Sundays at 02:00 UTC
8+
- cron: '0 2 * * 0'
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
trufflehog:
15+
runs-on: ubuntu-latest
16+
outputs:
17+
count: ${{ steps.count.outputs.findings }}
18+
steps:
19+
- name: Checkout code
20+
uses: actions/checkout@v4
21+
with:
22+
fetch-depth: 0
23+
24+
- name: Run TruffleHog OSS
25+
run: |
26+
# Use TruffleHog directly to capture JSON output properly
27+
docker run --rm -v "$(pwd):/pwd" \
28+
trufflesecurity/trufflehog:latest filesystem /pwd \
29+
--json --only-verified > trufflehog_output.json || true
30+
continue-on-error: true
31+
id: trufflehog
32+
33+
- name: Count TruffleHog findings
34+
id: count
35+
run: |
36+
# Count findings from TruffleHog output (it outputs JSON lines)
37+
count=0
38+
if [ -f trufflehog_output.json ]; then
39+
count=$(cat trufflehog_output.json | \
40+
grep -c "\"verified\":" || echo "0")
41+
fi
42+
echo "findings=$count" >> $GITHUB_OUTPUT
43+
echo "TruffleHog found $count verified secrets"
44+
45+
git-secrets:
46+
runs-on: ubuntu-latest
47+
outputs:
48+
count: ${{ steps.count.outputs.findings }}
49+
steps:
50+
- name: Checkout code
51+
uses: actions/checkout@v4
52+
53+
- name: Install git-secrets
54+
run: |
55+
git clone https://github.com/awslabs/git-secrets.git
56+
cd git-secrets
57+
sudo make install
58+
59+
- name: Initialize git-secrets
60+
run: |
61+
git secrets --register-aws
62+
git secrets --install
63+
64+
- name: Run git-secrets scan
65+
id: scan
66+
run: |
67+
set +e
68+
git secrets --scan > git_secrets_output.txt 2>&1
69+
exit_code=$?
70+
echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
71+
cat git_secrets_output.txt
72+
continue-on-error: true
73+
74+
- name: Count git-secrets findings
75+
id: count
76+
run: |
77+
count=0
78+
if [ -f git_secrets_output.txt ]; then
79+
# Count lines that indicate findings (exclude headers/empty lines)
80+
count=$(grep -c ".*:.*:.*" git_secrets_output.txt || echo "0")
81+
fi
82+
echo "findings=$count" >> $GITHUB_OUTPUT
83+
echo "git-secrets found $count secrets"
84+
85+
detect-secrets:
86+
runs-on: ubuntu-latest
87+
outputs:
88+
count: ${{ steps.count.outputs.findings }}
89+
steps:
90+
- name: Checkout code
91+
uses: actions/checkout@v4
92+
93+
- name: Set up Python
94+
uses: actions/setup-python@v4
95+
with:
96+
python-version: '3.11'
97+
98+
- name: Install detect-secrets
99+
run: |
100+
pip install detect-secrets
101+
102+
- name: Run detect-secrets scan
103+
run: |
104+
detect-secrets scan --all-files > detect_secrets_output.json
105+
continue-on-error: true
106+
107+
- name: Count detect-secrets findings
108+
id: count
109+
run: |
110+
count=0
111+
if [ -f detect_secrets_output.json ]; then
112+
# Count the number of potential secrets found
113+
count=$(jq '.results | to_entries | map(.value | length) | \
114+
add // 0' detect_secrets_output.json)
115+
fi
116+
echo "findings=$count" >> $GITHUB_OUTPUT
117+
echo "detect-secrets found $count potential secrets"
118+
119+
gitleaks:
120+
runs-on: ubuntu-latest
121+
outputs:
122+
count: ${{ steps.count.outputs.findings }}
123+
steps:
124+
- name: Checkout code
125+
uses: actions/checkout@v4
126+
with:
127+
fetch-depth: 0
128+
129+
- name: Install gitleaks
130+
run: |
131+
wget -O gitleaks.tar.gz \
132+
https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz
133+
tar -xf gitleaks.tar.gz
134+
sudo mv gitleaks /usr/local/bin/
135+
gitleaks version
136+
137+
- name: Run gitleaks scan
138+
run: |
139+
gitleaks detect --source . --report-format json \
140+
--report-path gitleaks_output.json --no-git || \
141+
echo "Gitleaks scan completed"
142+
continue-on-error: true
143+
144+
- name: Count gitleaks findings
145+
id: count
146+
run: |
147+
count=0
148+
if [ -f gitleaks_output.json ]; then
149+
# Count findings in JSON output
150+
count=$(jq 'length' gitleaks_output.json 2>/dev/null || echo "0")
151+
fi
152+
echo "findings=$count" >> $GITHUB_OUTPUT
153+
echo "gitleaks found $count secrets"
154+
155+
gittyleaks:
156+
runs-on: ubuntu-latest
157+
outputs:
158+
count: ${{ steps.count.outputs.findings }}
159+
steps:
160+
- name: Checkout code
161+
uses: actions/checkout@v4
162+
163+
- name: Set up Python
164+
uses: actions/setup-python@v4
165+
with:
166+
python-version: '3.11'
167+
168+
- name: Install gittyleaks
169+
run: |
170+
pip install gittyleaks
171+
172+
- name: Run gittyleaks scan
173+
run: |
174+
gittyleaks --find-anything > gittyleaks_output.txt 2>&1
175+
continue-on-error: true
176+
177+
- name: Count gittyleaks findings
178+
id: count
179+
run: |
180+
count=0
181+
if [ -f gittyleaks_output.txt ]; then
182+
# Count lines that contain findings (exclude header/footer lines)
183+
count=$(grep ":" gittyleaks_output.txt | \
184+
grep -v "Bot Detective" | grep -v "^---" | \
185+
wc -l || echo "0")
186+
fi
187+
echo "findings=$count" >> $GITHUB_OUTPUT
188+
echo "gittyleaks found $count secrets"
189+
190+
whispers:
191+
runs-on: ubuntu-latest
192+
outputs:
193+
count: ${{ steps.count.outputs.findings }}
194+
steps:
195+
- name: Checkout code
196+
uses: actions/checkout@v4
197+
198+
- name: Set up Python
199+
uses: actions/setup-python@v4
200+
with:
201+
python-version: '3.11'
202+
203+
- name: Install whispers (with timeout handling)
204+
run: |
205+
# Try to install whispers with a timeout and fallback
206+
timeout 300 pip install whispers || echo "Failed to install whispers"
207+
continue-on-error: true
208+
209+
- name: Run whispers scan
210+
run: |
211+
if command -v whispers >/dev/null 2>&1; then
212+
whispers . --output whispers_output.json --format json || \
213+
echo "Whispers scan failed"
214+
else
215+
echo "Whispers not available, skipping scan"
216+
echo "[]" > whispers_output.json
217+
fi
218+
continue-on-error: true
219+
220+
- name: Count whispers findings
221+
id: count
222+
run: |
223+
count=0
224+
if [ -f whispers_output.json ]; then
225+
# Count findings in JSON output
226+
count=$(jq 'length' whispers_output.json 2>/dev/null || echo "0")
227+
fi
228+
echo "findings=$count" >> $GITHUB_OUTPUT
229+
echo "whispers found $count secrets"
230+
231+
trufflehog3:
232+
runs-on: ubuntu-latest
233+
outputs:
234+
count: ${{ steps.count.outputs.findings }}
235+
steps:
236+
- name: Checkout code
237+
uses: actions/checkout@v4
238+
239+
- name: Set up Python
240+
uses: actions/setup-python@v4
241+
with:
242+
python-version: '3.11'
243+
244+
- name: Install trufflehog3
245+
run: |
246+
pip install trufflehog3
247+
248+
- name: Run trufflehog3 scan
249+
run: |
250+
trufflehog3 . --format json > trufflehog3_output.json 2>&1 || \
251+
echo "TruffleHog3 scan completed with warnings"
252+
continue-on-error: true
253+
254+
- name: Count trufflehog3 findings
255+
id: count
256+
run: |
257+
count=0
258+
if [ -f trufflehog3_output.json ]; then
259+
# Count findings - each line that starts with '{' is a finding
260+
count=$(grep -c '^{' trufflehog3_output.json || echo "0")
261+
fi
262+
echo "findings=$count" >> $GITHUB_OUTPUT
263+
echo "trufflehog3 found $count secrets"
264+
265+
summary:
266+
needs: [trufflehog, git-secrets, gitleaks, detect-secrets, gittyleaks,
267+
whispers, trufflehog3]
268+
runs-on: ubuntu-latest
269+
steps:
270+
- name: Create Summary Report
271+
run: |
272+
echo "# Secret Scanner Comparison Results" >> $GITHUB_STEP_SUMMARY
273+
echo "" >> $GITHUB_STEP_SUMMARY
274+
echo "| Scanner | Secrets Found |" >> $GITHUB_STEP_SUMMARY
275+
echo "|---------|---------------|" >> $GITHUB_STEP_SUMMARY
276+
echo "| TruffleHog | ${{ needs.trufflehog.outputs.count }} |" \
277+
>> $GITHUB_STEP_SUMMARY
278+
echo "| git-secrets | ${{ needs.git-secrets.outputs.count }} |" \
279+
>> $GITHUB_STEP_SUMMARY
280+
echo "| gitleaks | ${{ needs.gitleaks.outputs.count }} |" \
281+
>> $GITHUB_STEP_SUMMARY
282+
echo "| detect-secrets |" \
283+
"${{ needs.detect-secrets.outputs.count }} |" \
284+
>> $GITHUB_STEP_SUMMARY
285+
echo "| gittyleaks | ${{ needs.gittyleaks.outputs.count }} |" \
286+
>> $GITHUB_STEP_SUMMARY
287+
echo "| whispers | ${{ needs.whispers.outputs.count }} |" \
288+
>> $GITHUB_STEP_SUMMARY
289+
echo "| trufflehog3 | ${{ needs.trufflehog3.outputs.count }} |" \
290+
>> $GITHUB_STEP_SUMMARY
291+
echo "" >> $GITHUB_STEP_SUMMARY
292+
echo "**Total unique scanning tools tested:** 7" \
293+
>> $GITHUB_STEP_SUMMARY
294+
echo "" >> $GITHUB_STEP_SUMMARY
295+
echo "_This benchmark helps understand the relative effectiveness" \
296+
"of different secret scanning tools on the OWASP WrongSecrets" \
297+
"repository._" >> $GITHUB_STEP_SUMMARY
298+
299+
# Also output to console
300+
echo "=== Secret Scanner Comparison Results ==="
301+
echo "TruffleHog: ${{ needs.trufflehog.outputs.count }} secrets"
302+
echo "git-secrets: ${{ needs.git-secrets.outputs.count }} secrets"
303+
echo "gitleaks: ${{ needs.gitleaks.outputs.count }} secrets"
304+
echo "detect-secrets: ${{ needs.detect-secrets.outputs.count }} \
305+
secrets"
306+
echo "gittyleaks: ${{ needs.gittyleaks.outputs.count }} secrets"
307+
echo "whispers: ${{ needs.whispers.outputs.count }} secrets"
308+
echo "trufflehog3: ${{ needs.trufflehog3.outputs.count }} secrets"

‎.gitignore‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
*.rar
2323
*.iml
2424

25-
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
25+
# virtual machine crash logs
2626
hs_err_pid*
2727

2828
.DS_Store
@@ -108,3 +108,12 @@ scripts/sort_contibutors/contributors_file.*
108108
# Cha-llen-ge 5-4
109109
# Se-cret en-cry-pted : h0vRDUS3VDN2gwfd1oftcMtD/YlKE0YfHLTvyk/lrY4=
110110
# K-E-Y : 6b3b913c09d62238b9b8c0cc78904e7e1f9a99fd0a04ebae50a8a9881d452bbd
111+
112+
# Scanner output files
113+
*secrets*output*.json
114+
*trufflehog*output*.json
115+
*gitleaks*output*.txt
116+
*gittyleaks*output*.txt
117+
*whispers*output*.json
118+
git_secrets_output.txt
119+
.secrets.baseline.json

‎README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,20 @@ As tons of secret detection tools are coming up for both Docker and Git, we are
475475
Want to know if your tool detects everything? We will keep track of the embedded secrets in [this issue](https://github.com/OWASP/wrongsecrets/issues/201) and have a [branch](https://github.com/OWASP/wrongsecrets/tree/experiment-bed) in which we put additional secrets for your tool to detect.
476476
The branch will contain a Docker container generation script using which you can eventually test your container secret scanning.
477477

478+
### Automated Secret Scanner Comparison
479+
480+
We now provide an automated GitHub Action workflow that benchmarks multiple secret scanning tools against the WrongSecrets codebase. The [Secret Scanner Comparison workflow](/.github/workflows/scanner-comparison.yml) tests 7 different tools:
481+
482+
- **TruffleHog** - Docker-based secret scanner
483+
- **git-secrets** - AWS Labs' git hook scanner
484+
- **gitleaks** - High-performance Go-based scanner
485+
- **detect-secrets** - Yelp's enterprise scanner
486+
- **gittyleaks** - Python-based pattern detector
487+
- **whispers** - Skyscanner's structured scanner
488+
- **trufflehog3** - Python version of TruffleHog
489+
490+
The workflow runs weekly and provides a comparison table showing how many secrets each tool detects, helping you understand the relative effectiveness of different secret scanning tools. See [docs/scanner-comparison.md](/docs/scanner-comparison.md) for more details on running and interpreting the results.
491+
478492
## CTF
479493

480494
We have 3 ways of playing CTFs:

0 commit comments

Comments
 (0)