1+ ---
12name : Secret Scanner Comparison Benchmark
23
34on :
@@ -19,21 +20,24 @@ jobs:
1920 uses : actions/checkout@v4
2021 with :
2122 fetch-depth : 0
22-
23+
2324 - name : Run TruffleHog OSS
2425 run : |
2526 # Use TruffleHog directly to capture JSON output properly
26- docker run --rm -v "$(pwd):/pwd" trufflesecurity/trufflehog:latest filesystem /pwd --json --only-verified > trufflehog_output.json || true
27+ docker run --rm -v "$(pwd):/pwd" \
28+ trufflesecurity/trufflehog:latest filesystem /pwd \
29+ --json --only-verified > trufflehog_output.json || true
2730 continue-on-error : true
2831 id : trufflehog
29-
32+
3033 - name : Count TruffleHog findings
3134 id : count
3235 run : |
3336 # Count findings from TruffleHog output (it outputs JSON lines)
3437 count=0
3538 if [ -f trufflehog_output.json ]; then
36- count=$(cat trufflehog_output.json | grep -c "\"verified\":" || echo "0")
39+ count=$(cat trufflehog_output.json | \
40+ grep -c "\"verified\":" || echo "0")
3741 fi
3842 echo "findings=$count" >> $GITHUB_OUTPUT
3943 echo "TruffleHog found $count verified secrets"
@@ -45,18 +49,18 @@ jobs:
4549 steps :
4650 - name : Checkout code
4751 uses : actions/checkout@v4
48-
52+
4953 - name : Install git-secrets
5054 run : |
5155 git clone https://github.com/awslabs/git-secrets.git
5256 cd git-secrets
5357 sudo make install
54-
58+
5559 - name : Initialize git-secrets
5660 run : |
5761 git secrets --register-aws
5862 git secrets --install
59-
63+
6064 - name : Run git-secrets scan
6165 id : scan
6266 run : |
6670 echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
6771 cat git_secrets_output.txt
6872 continue-on-error : true
69-
73+
7074 - name : Count git-secrets findings
7175 id : count
7276 run : |
@@ -85,28 +89,29 @@ jobs:
8589 steps :
8690 - name : Checkout code
8791 uses : actions/checkout@v4
88-
92+
8993 - name : Set up Python
9094 uses : actions/setup-python@v4
9195 with :
9296 python-version : ' 3.11'
93-
97+
9498 - name : Install detect-secrets
9599 run : |
96100 pip install detect-secrets
97-
101+
98102 - name : Run detect-secrets scan
99103 run : |
100104 detect-secrets scan --all-files > detect_secrets_output.json
101105 continue-on-error : true
102-
106+
103107 - name : Count detect-secrets findings
104108 id : count
105109 run : |
106110 count=0
107111 if [ -f detect_secrets_output.json ]; then
108112 # Count the number of potential secrets found
109- count=$(jq '.results | to_entries | map(.value | length) | add // 0' detect_secrets_output.json)
113+ count=$(jq '.results | to_entries | map(.value | length) | \
114+ add // 0' detect_secrets_output.json)
110115 fi
111116 echo "findings=$count" >> $GITHUB_OUTPUT
112117 echo "detect-secrets found $count potential secrets"
@@ -120,19 +125,22 @@ jobs:
120125 uses : actions/checkout@v4
121126 with :
122127 fetch-depth : 0
123-
128+
124129 - name : Install gitleaks
125130 run : |
126- wget -O gitleaks.tar.gz https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz
131+ wget -O gitleaks.tar.gz \
132+ https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz
127133 tar -xf gitleaks.tar.gz
128134 sudo mv gitleaks /usr/local/bin/
129135 gitleaks version
130-
136+
131137 - name : Run gitleaks scan
132138 run : |
133- gitleaks detect --source . --report-format json --report-path gitleaks_output.json --no-git || echo "Gitleaks scan completed"
139+ gitleaks detect --source . --report-format json \
140+ --report-path gitleaks_output.json --no-git || \
141+ echo "Gitleaks scan completed"
134142 continue-on-error : true
135-
143+
136144 - name : Count gitleaks findings
137145 id : count
138146 run : |
@@ -151,28 +159,30 @@ jobs:
151159 steps :
152160 - name : Checkout code
153161 uses : actions/checkout@v4
154-
162+
155163 - name : Set up Python
156164 uses : actions/setup-python@v4
157165 with :
158166 python-version : ' 3.11'
159-
167+
160168 - name : Install gittyleaks
161169 run : |
162170 pip install gittyleaks
163-
171+
164172 - name : Run gittyleaks scan
165173 run : |
166174 gittyleaks --find-anything > gittyleaks_output.txt 2>&1
167175 continue-on-error : true
168-
176+
169177 - name : Count gittyleaks findings
170178 id : count
171179 run : |
172180 count=0
173181 if [ -f gittyleaks_output.txt ]; then
174182 # Count lines that contain findings (exclude header/footer lines)
175- count=$(grep ":" gittyleaks_output.txt | grep -v "Bot Detective" | grep -v "^---" | wc -l || echo "0")
183+ count=$(grep ":" gittyleaks_output.txt | \
184+ grep -v "Bot Detective" | grep -v "^---" | \
185+ wc -l || echo "0")
176186 fi
177187 echo "findings=$count" >> $GITHUB_OUTPUT
178188 echo "gittyleaks found $count secrets"
@@ -184,28 +194,29 @@ jobs:
184194 steps :
185195 - name : Checkout code
186196 uses : actions/checkout@v4
187-
197+
188198 - name : Set up Python
189199 uses : actions/setup-python@v4
190200 with :
191201 python-version : ' 3.11'
192-
202+
193203 - name : Install whispers (with timeout handling)
194204 run : |
195205 # Try to install whispers with a timeout and fallback
196206 timeout 300 pip install whispers || echo "Failed to install whispers"
197207 continue-on-error : true
198-
208+
199209 - name : Run whispers scan
200210 run : |
201211 if command -v whispers >/dev/null 2>&1; then
202- whispers . --output whispers_output.json --format json || echo "Whispers scan failed"
212+ whispers . --output whispers_output.json --format json || \
213+ echo "Whispers scan failed"
203214 else
204215 echo "Whispers not available, skipping scan"
205216 echo "[]" > whispers_output.json
206217 fi
207218 continue-on-error : true
208-
219+
209220 - name : Count whispers findings
210221 id : count
211222 run : |
@@ -224,21 +235,22 @@ jobs:
224235 steps :
225236 - name : Checkout code
226237 uses : actions/checkout@v4
227-
238+
228239 - name : Set up Python
229240 uses : actions/setup-python@v4
230241 with :
231242 python-version : ' 3.11'
232-
243+
233244 - name : Install trufflehog3
234245 run : |
235246 pip install trufflehog3
236-
247+
237248 - name : Run trufflehog3 scan
238249 run : |
239- trufflehog3 . --format json > trufflehog3_output.json 2>&1 || echo "TruffleHog3 scan completed with warnings"
250+ trufflehog3 . --format json > trufflehog3_output.json 2>&1 || \
251+ echo "TruffleHog3 scan completed with warnings"
240252 continue-on-error : true
241-
253+
242254 - name : Count trufflehog3 findings
243255 id : count
244256 run : |
@@ -251,7 +263,8 @@ jobs:
251263 echo "trufflehog3 found $count secrets"
252264
253265 summary :
254- needs : [trufflehog, git-secrets, gitleaks, detect-secrets, gittyleaks, whispers, trufflehog3]
266+ needs : [trufflehog, git-secrets, gitleaks, detect-secrets, gittyleaks,
267+ whispers, trufflehog3]
255268 runs-on : ubuntu-latest
256269 steps :
257270 - name : Create Summary Report
@@ -260,24 +273,36 @@ jobs:
260273 echo "" >> $GITHUB_STEP_SUMMARY
261274 echo "| Scanner | Secrets Found |" >> $GITHUB_STEP_SUMMARY
262275 echo "|---------|---------------|" >> $GITHUB_STEP_SUMMARY
263- echo "| TruffleHog | ${{ needs.trufflehog.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
264- echo "| git-secrets | ${{ needs.git-secrets.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
265- echo "| gitleaks | ${{ needs.gitleaks.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
266- echo "| detect-secrets | ${{ needs.detect-secrets.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
267- echo "| gittyleaks | ${{ needs.gittyleaks.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
268- echo "| whispers | ${{ needs.whispers.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
269- echo "| trufflehog3 | ${{ needs.trufflehog3.outputs.count }} |" >> $GITHUB_STEP_SUMMARY
276+ echo "| TruffleHog | ${{ needs.trufflehog.outputs.count }} |" \
277+ >> $GITHUB_STEP_SUMMARY
278+ echo "| git-secrets | ${{ needs.git-secrets.outputs.count }} |" \
279+ >> $GITHUB_STEP_SUMMARY
280+ echo "| gitleaks | ${{ needs.gitleaks.outputs.count }} |" \
281+ >> $GITHUB_STEP_SUMMARY
282+ echo "| detect-secrets |" \
283+ "${{ needs.detect-secrets.outputs.count }} |" \
284+ >> $GITHUB_STEP_SUMMARY
285+ echo "| gittyleaks | ${{ needs.gittyleaks.outputs.count }} |" \
286+ >> $GITHUB_STEP_SUMMARY
287+ echo "| whispers | ${{ needs.whispers.outputs.count }} |" \
288+ >> $GITHUB_STEP_SUMMARY
289+ echo "| trufflehog3 | ${{ needs.trufflehog3.outputs.count }} |" \
290+ >> $GITHUB_STEP_SUMMARY
270291 echo "" >> $GITHUB_STEP_SUMMARY
271- echo "**Total unique scanning tools tested:** 7" >> $GITHUB_STEP_SUMMARY
292+ echo "**Total unique scanning tools tested:** 7" \
293+ >> $GITHUB_STEP_SUMMARY
272294 echo "" >> $GITHUB_STEP_SUMMARY
273- echo "_This benchmark helps understand the relative effectiveness of different secret scanning tools on the OWASP WrongSecrets repository._" >> $GITHUB_STEP_SUMMARY
274-
295+ echo "_This benchmark helps understand the relative effectiveness" \
296+ "of different secret scanning tools on the OWASP WrongSecrets" \
297+ "repository._" >> $GITHUB_STEP_SUMMARY
298+
275299 # Also output to console
276300 echo "=== Secret Scanner Comparison Results ==="
277301 echo "TruffleHog: ${{ needs.trufflehog.outputs.count }} secrets"
278302 echo "git-secrets: ${{ needs.git-secrets.outputs.count }} secrets"
279303 echo "gitleaks: ${{ needs.gitleaks.outputs.count }} secrets"
280- echo "detect-secrets: ${{ needs.detect-secrets.outputs.count }} secrets"
304+ echo "detect-secrets: ${{ needs.detect-secrets.outputs.count }} \
305+ secrets"
281306 echo "gittyleaks: ${{ needs.gittyleaks.outputs.count }} secrets"
282307 echo "whispers: ${{ needs.whispers.outputs.count }} secrets"
283- echo "trufflehog3: ${{ needs.trufflehog3.outputs.count }} secrets"
308+ echo "trufflehog3: ${{ needs.trufflehog3.outputs.count }} secrets"
0 commit comments