Skip to content

Commit 8bab169

Browse files
Copilotcommjoen
Copilot
and
commjoen
committed
Add comprehensive secret scanner comparison workflow
Co-authored-by: commjoen <[email protected]>
1 parent 27a6cef commit 8bab169

File tree

2 files changed

+1179
-0
lines changed

2 files changed

+1179
-0
lines changed

‎.github/workflows/scanner-comparison.yml‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
name: Secret Scanner Comparison Benchmark
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
# Run weekly on Sundays at 02:00 UTC
7+
- cron: '0 2 * * 0'
8+
9+
permissions:
10+
contents: read
11+
12+
jobs:
13+
trufflehog:
14+
runs-on: ubuntu-latest
15+
outputs:
16+
count: ${{ steps.count.outputs.findings }}
17+
steps:
18+
- name: Checkout code
19+
uses: actions/checkout@v4
20+
with:
21+
fetch-depth: 0
22+
23+
- name: Run TruffleHog OSS
24+
run: |
25+
# Use TruffleHog directly to capture JSON output properly
26+
docker run --rm -v "$(pwd):/pwd" trufflesecurity/trufflehog:latest filesystem /pwd --json --only-verified > trufflehog_output.json || true
27+
continue-on-error: true
28+
id: trufflehog
29+
30+
- name: Count TruffleHog findings
31+
id: count
32+
run: |
33+
# Count findings from TruffleHog output (it outputs JSON lines)
34+
count=0
35+
if [ -f trufflehog_output.json ]; then
36+
count=$(cat trufflehog_output.json | grep -c "\"verified\":" || echo "0")
37+
fi
38+
echo "findings=$count" >> $GITHUB_OUTPUT
39+
echo "TruffleHog found $count verified secrets"
40+
41+
git-secrets:
42+
runs-on: ubuntu-latest
43+
outputs:
44+
count: ${{ steps.count.outputs.findings }}
45+
steps:
46+
- name: Checkout code
47+
uses: actions/checkout@v4
48+
49+
- name: Install git-secrets
50+
run: |
51+
git clone https://github.com/awslabs/git-secrets.git
52+
cd git-secrets
53+
sudo make install
54+
55+
- name: Initialize git-secrets
56+
run: |
57+
git secrets --register-aws
58+
git secrets --install
59+
60+
- name: Run git-secrets scan
61+
id: scan
62+
run: |
63+
set +e
64+
git secrets --scan > git_secrets_output.txt 2>&1
65+
exit_code=$?
66+
echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
67+
cat git_secrets_output.txt
68+
continue-on-error: true
69+
70+
- name: Count git-secrets findings
71+
id: count
72+
run: |
73+
count=0
74+
if [ -f git_secrets_output.txt ]; then
75+
# Count lines that indicate findings (exclude headers/empty lines)
76+
count=$(grep -c ".*:.*:.*" git_secrets_output.txt || echo "0")
77+
fi
78+
echo "findings=$count" >> $GITHUB_OUTPUT
79+
echo "git-secrets found $count secrets"
80+
81+
detect-secrets:
82+
runs-on: ubuntu-latest
83+
outputs:
84+
count: ${{ steps.count.outputs.findings }}
85+
steps:
86+
- name: Checkout code
87+
uses: actions/checkout@v4
88+
89+
- name: Set up Python
90+
uses: actions/setup-python@v4
91+
with:
92+
python-version: '3.11'
93+
94+
- name: Install detect-secrets
95+
run: |
96+
pip install detect-secrets
97+
98+
- name: Run detect-secrets scan
99+
run: |
100+
detect-secrets scan --all-files > detect_secrets_output.json
101+
continue-on-error: true
102+
103+
- name: Count detect-secrets findings
104+
id: count
105+
run: |
106+
count=0
107+
if [ -f detect_secrets_output.json ]; then
108+
# Count the number of potential secrets found
109+
count=$(jq '.results | to_entries | map(.value | length) | add // 0' detect_secrets_output.json)
110+
fi
111+
echo "findings=$count" >> $GITHUB_OUTPUT
112+
echo "detect-secrets found $count potential secrets"
113+
114+
gittyleaks:
115+
runs-on: ubuntu-latest
116+
outputs:
117+
count: ${{ steps.count.outputs.findings }}
118+
steps:
119+
- name: Checkout code
120+
uses: actions/checkout@v4
121+
122+
- name: Set up Python
123+
uses: actions/setup-python@v4
124+
with:
125+
python-version: '3.11'
126+
127+
- name: Install gittyleaks
128+
run: |
129+
pip install gittyleaks
130+
131+
- name: Run gittyleaks scan
132+
run: |
133+
gittyleaks --find-anything > gittyleaks_output.txt 2>&1
134+
continue-on-error: true
135+
136+
- name: Count gittyleaks findings
137+
id: count
138+
run: |
139+
count=0
140+
if [ -f gittyleaks_output.txt ]; then
141+
# Count lines that contain findings
142+
count=$(grep -c "Found" gittyleaks_output.txt || echo "0")
143+
fi
144+
echo "findings=$count" >> $GITHUB_OUTPUT
145+
echo "gittyleaks found $count secrets"
146+
147+
whispers:
148+
runs-on: ubuntu-latest
149+
outputs:
150+
count: ${{ steps.count.outputs.findings }}
151+
steps:
152+
- name: Checkout code
153+
uses: actions/checkout@v4
154+
155+
- name: Set up Python
156+
uses: actions/setup-python@v4
157+
with:
158+
python-version: '3.11'
159+
160+
- name: Install whispers
161+
run: |
162+
pip install whispers
163+
164+
- name: Run whispers scan
165+
run: |
166+
whispers . --output whispers_output.json --format json
167+
continue-on-error: true
168+
169+
- name: Count whispers findings
170+
id: count
171+
run: |
172+
count=0
173+
if [ -f whispers_output.json ]; then
174+
# Count findings in JSON output
175+
count=$(jq 'length' whispers_output.json || echo "0")
176+
fi
177+
echo "findings=$count" >> $GITHUB_OUTPUT
178+
echo "whispers found $count secrets"
179+
180+
trufflehog3:
181+
runs-on: ubuntu-latest
182+
outputs:
183+
count: ${{ steps.count.outputs.findings }}
184+
steps:
185+
- name: Checkout code
186+
uses: actions/checkout@v4
187+
188+
- name: Set up Python
189+
uses: actions/setup-python@v4
190+
with:
191+
python-version: '3.11'
192+
193+
- name: Install trufflehog3
194+
run: |
195+
pip install trufflehog3
196+
197+
- name: Run trufflehog3 scan
198+
run: |
199+
trufflehog3 . --format json > trufflehog3_output.json 2>&1
200+
continue-on-error: true
201+
202+
- name: Count trufflehog3 findings
203+
id: count
204+
run: |
205+
count=0
206+
if [ -f trufflehog3_output.json ]; then
207+
# Count JSON objects (each line is a finding)
208+
count=$(cat trufflehog3_output.json | jq -s 'length' || echo "0")
209+
fi
210+
echo "findings=$count" >> $GITHUB_OUTPUT
211+
echo "trufflehog3 found $count secrets"
212+
213+
summary:
214+
needs: [trufflehog, git-secrets, detect-secrets, gittyleaks, whispers, trufflehog3]
215+
runs-on: ubuntu-latest
216+
steps:
217+
- name: Create Summary Report
218+
run: |
219+
echo "# Secret Scanner Comparison Results" >> $GITHUB_STEP_SUMMARY
220+
echo "" >> $GITHUB_STEP_SUMMARY
221+
echo "| Scanner | Secrets Found |" >> $GITHUB_STEP_SUMMARY
222+
echo "|---------|---------------|" >> $GITHUB_STEP_SUMMARY
223+
echo "| TruffleHog | ${{ needs.trufflehog.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
224+
echo "| git-secrets | ${{ needs.git-secrets.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
225+
echo "| detect-secrets | ${{ needs.detect-secrets.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
226+
echo "| gittyleaks | ${{ needs.gittyleaks.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
227+
echo "| whispers | ${{ needs.whispers.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
228+
echo "| trufflehog3 | ${{ needs.trufflehog3.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
229+
echo "" >> $GITHUB_STEP_SUMMARY
230+
echo "**Total unique scanning tools tested:** 6" >> $GITHUB_STEP_SUMMARY
231+
echo "" >> $GITHUB_STEP_SUMMARY
232+
echo "_This benchmark helps understand the relative effectiveness of different secret scanning tools on the OWASP WrongSecrets repository._" >> $GITHUB_STEP_SUMMARY
233+
234+
# Also output to console
235+
echo "=== Secret Scanner Comparison Results ==="
236+
echo "TruffleHog: ${{ needs.trufflehog.outputs.count }} secrets"
237+
echo "git-secrets: ${{ needs.git-secrets.outputs.count }} secrets"
238+
echo "detect-secrets: ${{ needs.detect-secrets.outputs.count }} secrets"
239+
echo "gittyleaks: ${{ needs.gittyleaks.outputs.count }} secrets"
240+
echo "whispers: ${{ needs.whispers.outputs.count }} secrets"
241+
echo "trufflehog3: ${{ needs.trufflehog3.outputs.count }} secrets"

0 commit comments

Comments
 (0)