Skip to content

Commit a2f5c34

Browse files
davewichersdavewichers
committed
Determine the number of test cases per CWE in per CategoryGroup, if
enabled, and add a chart at the bottom of the CategoryGroup page which lists the Test Case Counts for each CWE that have test cases in that CategoryGroup.
1 parent 4c68dbd commit a2f5c34

File tree

6 files changed

+166
-30
lines changed

6 files changed

+166
-30
lines changed

‎plugin/src/main/java/org/owasp/benchmarkutils/helpers/CategoryGroup.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import java.util.HashMap;
2121
import java.util.Map;
22+
import java.util.Set;
2223

2324
/*
2425
* This class contains a single vulnerability group. Each group has a set of associated CWEs, so the entire group can be scored in the same manner that an individual vulnerability category can be scored.
@@ -84,6 +85,15 @@ public CWE getCWE(int cweNum) {
8485
return this.cweToCategoryGroupMap.get(cweNum);
8586
}
8687

88+
/**
89+
* Gets the list of all CWEs in this Category Group.
90+
*
91+
* @return The list of all CWEs in this Category Group.
92+
*/
93+
public Set<Integer> getCWEs() {
94+
return this.cweToCategoryGroupMap.keySet();
95+
}
96+
8797
public String toString() {
8898
return this.longname + "::" + this.abbreviation;
8999
}

‎plugin/src/main/java/org/owasp/benchmarkutils/score/BenchmarkScore.java‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import java.util.Map;
3636
import java.util.Optional;
3737
import java.util.Set;
38+
import java.util.SortedMap;
3839
import java.util.TreeMap;
3940
import java.util.TreeSet;
4041
import org.apache.commons.io.FileUtils;
@@ -45,6 +46,7 @@
4546
import org.apache.maven.plugins.annotations.Parameter;
4647
import org.owasp.benchmarkutils.helpers.Categories;
4748
import org.owasp.benchmarkutils.helpers.Category;
49+
import org.owasp.benchmarkutils.helpers.CategoryGroup;
4850
import org.owasp.benchmarkutils.helpers.CategoryGroups;
4951
import org.owasp.benchmarkutils.helpers.Utils;
5052
import org.owasp.benchmarkutils.score.domain.TestSuiteName;
@@ -53,10 +55,12 @@
5355
import org.owasp.benchmarkutils.score.report.ScatterInterpretation;
5456
import org.owasp.benchmarkutils.score.report.ScatterVulns;
5557
import org.owasp.benchmarkutils.score.report.html.CommercialAveragesTable;
58+
import org.owasp.benchmarkutils.score.report.html.HtmlStringBuilder;
5659
import org.owasp.benchmarkutils.score.report.html.MenuUpdater;
5760
import org.owasp.benchmarkutils.score.report.html.OverallStatsTable;
5861
import org.owasp.benchmarkutils.score.report.html.ToolScorecard;
5962
import org.owasp.benchmarkutils.score.report.html.VulnerabilityStatsTable;
63+
import org.owasp.benchmarkutils.score.service.CountsPerCWE;
6064
import org.owasp.benchmarkutils.score.service.ExpectedResultsProvider;
6165
import org.owasp.benchmarkutils.score.service.ResultsFileCreator;
6266

@@ -505,8 +509,6 @@ public static void main(String[] args) {
505509
TESTSUITENAME,
506510
TESTSUITEVERSION,
507511
commercialAveragesTable,
508-
// commercialCategoryGroupAveragesTable, - DRW TODO - Needed? Have to turn
509-
// on commercial averages to test.
510512
tools,
511513
vulnSet,
512514
catGroupsSet,
@@ -1080,6 +1082,18 @@ private static void generateVulnerabilityScorecards(
10801082
config.report.html.precisionKeyEntry
10811083
+ config.report.html.fsCoreEntry);
10821084

1085+
// Add optional details of test cases per CWE included in a Category Group
1086+
html =
1087+
html.replace(
1088+
"${CategoryGroupDetailsTitle}",
1089+
(useCategoryGroups
1090+
? "<h2>Test Case Counts for CWEs Included in this Group</h2>"
1091+
: ""));
1092+
html =
1093+
html.replace(
1094+
"${CategoryGroupDetailsTable}",
1095+
generateCategoryGroupDetailsTable(cat, useCategoryGroups));
1096+
10831097
Files.write(htmlFile.toPath(), html.getBytes());
10841098

10851099
// Only build commercial stats scorecard if there are 2+ commercial tools
@@ -1124,4 +1138,67 @@ private static void generateVulnerabilityScorecards(
11241138
}
11251139
} // end if commercialAveragesTable.hasEntries()
11261140
}
1141+
1142+
private static String generateCategoryGroupDetailsTable(
1143+
String categoryGroup, boolean useCategoryGroups) {
1144+
if (!useCategoryGroups) return "";
1145+
HtmlStringBuilder htmlBuilder = new HtmlStringBuilder();
1146+
1147+
htmlBuilder.beginTable("table");
1148+
1149+
htmlBuilder.beginTr();
1150+
htmlBuilder.th("CWE");
1151+
htmlBuilder.th("Vulnerability Category");
1152+
htmlBuilder.th("TPs");
1153+
htmlBuilder.th("FPs");
1154+
htmlBuilder.th("Total");
1155+
htmlBuilder.endTr();
1156+
1157+
CategoryGroup currentGroup = CategoryGroups.getCategoryGroupByName(categoryGroup);
1158+
Set<Integer> cweList = currentGroup.getCWEs();
1159+
1160+
// Create a sorted list by Vuln Category name (e.g., Hard-coded Password) where there are
1161+
// testcases in a CWE in this CategoryGroup
1162+
SortedMap<String, Integer> sortedCWEList = new TreeMap<>();
1163+
for (int cwe : cweList) {
1164+
CountsPerCWE cweCounts = ExpectedResultsProvider.getTestcaseCountsForCWE(cwe);
1165+
if (cweCounts != null) {
1166+
Category cat = Categories.getCategoryByCWE(cwe);
1167+
sortedCWEList.put(cat.getName(), Integer.valueOf(cwe));
1168+
}
1169+
}
1170+
1171+
// Loop through sortedCWEList and add a row for each CWE with these details to the table
1172+
int totalTpCount = 0, totalFpCount = 0, totalCount = 0;
1173+
for (String vulnType : sortedCWEList.keySet()) {
1174+
Integer CWE = sortedCWEList.get(vulnType);
1175+
CountsPerCWE cweCounts = ExpectedResultsProvider.getTestcaseCountsForCWE(CWE);
1176+
htmlBuilder.beginTr();
1177+
htmlBuilder.td(CWE);
1178+
htmlBuilder.td(vulnType);
1179+
int tpCountForCWE = cweCounts.getTPCount();
1180+
totalTpCount += tpCountForCWE;
1181+
htmlBuilder.td(tpCountForCWE);
1182+
int fpCountForCWE = cweCounts.getFPCount();
1183+
totalFpCount += fpCountForCWE;
1184+
htmlBuilder.td(fpCountForCWE);
1185+
int countTotalForCWE = tpCountForCWE + fpCountForCWE;
1186+
totalCount += countTotalForCWE;
1187+
htmlBuilder.td(countTotalForCWE);
1188+
htmlBuilder.endTr();
1189+
}
1190+
1191+
// And final total row
1192+
htmlBuilder.beginTr();
1193+
htmlBuilder.td("<b>Grand Total</b>");
1194+
htmlBuilder.td("");
1195+
htmlBuilder.td("<b>" + totalTpCount + "</b>");
1196+
htmlBuilder.td("<b>" + totalFpCount + "</b>");
1197+
htmlBuilder.td("<b>" + totalCount + "</b>");
1198+
htmlBuilder.endTr();
1199+
1200+
htmlBuilder.endTable();
1201+
1202+
return htmlBuilder.toString();
1203+
}
11271204
}

‎plugin/src/main/java/org/owasp/benchmarkutils/score/TestCaseResult.java‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -260,22 +260,6 @@ public CategoryGroup getCategoryGroup() {
260260
return this.categoryGroup;
261261
}
262262

263-
/**
264-
* This method is used to abstract away how filenames are matched against a test case, that way
265-
* it can be enhanced to support different test suite formats, and the logic on how to do this
266-
* isn't implemented in every individual parser. This method expects that
267-
* ExpectedResultsProvider.getExpectedResults().isTestCaseFile(filename) was called first to
268-
* verify this file is a test case in the test suite being scored. It sets the CWE number and
269-
* the test case name in this TestCaseResult if successful. It halts with an error if the
270-
* supplied filename is not a valid test case.
271-
*
272-
* @param cwe The CWE # reported by this tool.
273-
* @param filename The filename that might be a test case.
274-
* <p>public void setCWEAndTestCaseID(int cwe, String filename) { if
275-
* (ExpectedResultsProvider.getExpectedResults().isTestCaseFile(filename)) { // DRW FIXME:TODO -
276-
* Not implemented yet. Maybe just delete? } }
277-
*/
278-
279263
/**
280264
* The CWE category name, e.g., pathtraver, hash, cmdi, etc.
281265
*
@@ -349,9 +333,5 @@ public String toString() {
349333
+ getCWE()
350334
+ ", toolPassed: "
351335
+ isPassed();
352-
/* + ", evidence: "
353-
+ getEvidence()
354-
+ ", confidence: "
355-
+ getConfidence();
356-
*/ }
336+
}
357337
}

‎plugin/src/main/java/org/owasp/benchmarkutils/score/service/CountsPerCWE.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/**
2+
* OWASP Benchmark Project
3+
*
4+
* <p>This file is part of the Open Web Application Security Project (OWASP) Benchmark Project For
5+
* details, please see <a
6+
* href="https://owasp.org/www-project-benchmark/">https://owasp.org/www-project-benchmark/</a>.
7+
*
8+
* <p>The OWASP Benchmark is free software: you can redistribute it and/or modify it under the terms
9+
* of the GNU General Public License as published by the Free Software Foundation, version 2.
10+
*
11+
* <p>The OWASP Benchmark is distributed in the hope that it will be useful, but WITHOUT ANY
12+
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
13+
* PURPOSE. See the GNU General Public License for more details.
14+
*
15+
* @author Dave Wichers
16+
* @created 2025
17+
*/
18+
19+
/**
20+
* Instances of this class are used to keep track of the number True and False Positives for a CWE
21+
* in the expectedresults file.
22+
*/
23+
package org.owasp.benchmarkutils.score.service;
24+
25+
public class CountsPerCWE {
26+
27+
int truePositiveCount;
28+
int falsePositiveCount;
29+
30+
public int getTPCount() {
31+
return truePositiveCount;
32+
}
33+
34+
public int getFPCount() {
35+
return truePositiveCount;
36+
}
37+
}

‎plugin/src/main/java/org/owasp/benchmarkutils/score/service/ExpectedResultsProvider.java‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,14 @@
2121
import static java.lang.Integer.parseInt;
2222

2323
import java.io.IOException;
24+
import java.util.HashMap;
2425
import java.util.List;
26+
import java.util.Map;
2527
import java.util.Optional;
2628
import org.apache.commons.csv.CSVParser;
2729
import org.apache.commons.csv.CSVRecord;
2830
import org.owasp.benchmarkutils.helpers.Categories;
31+
import org.owasp.benchmarkutils.helpers.CategoryGroups;
2932
import org.owasp.benchmarkutils.score.BenchmarkScore;
3033
import org.owasp.benchmarkutils.score.ResultFile;
3134
import org.owasp.benchmarkutils.score.TestCaseResult;
@@ -49,6 +52,11 @@ public class ExpectedResultsProvider {
4952
private static boolean standardBenchmarkStyleScoring;
5053
private static TestSuiteResults expectedResults;
5154

55+
// If Category Groups are enabled, this map keeps counts of the # of TP and FP test cases per
56+
// CWE
57+
private static final Map<String, CountsPerCWE> testcaseCountsPerCWE =
58+
new HashMap<String, CountsPerCWE>();
59+
5260
public static TestSuiteResults parse(ResultFile resultFile) throws IOException {
5361
TestSuiteResults tr = new TestSuiteResults("Expected", true, null);
5462

@@ -57,9 +65,6 @@ public static TestSuiteResults parse(ResultFile resultFile) throws IOException {
5765
List<CSVRecord> allExpectedResults = parser.getRecords();
5866

5967
CSVRecord firstRecord = allExpectedResults.get(0);
60-
// setExpectedResultsMetadata() has side effect of setting
61-
// BenchmarkScore.TESTSUITENAME, BenchmarkScore.TESTCASENAME, and
62-
// ExpectedResultsProvider.standardBenchmarkStyleScoring
6368
setExpectedResultsMetadata(parser, firstRecord, tr);
6469

6570
// Parse all the expected results
@@ -68,11 +73,11 @@ public static TestSuiteResults parse(ResultFile resultFile) throws IOException {
6873

6974
String testCaseFileName = record.get(TEST_NAME);
7075
tcr.setTestCaseName(testCaseFileName);
71-
// tcr.setCategory(record.get(CATEGORY));
72-
tcr.setTruePositive(parseBoolean(record.get(REAL_VULNERABILITY)));
73-
int cwe = parseInt(record.get(CWE));
76+
boolean truePositive = parseBoolean(record.get(REAL_VULNERABILITY));
77+
tcr.setTruePositive(truePositive);
78+
String cweNumber = record.get(CWE);
79+
int cwe = parseInt(cweNumber);
7480
tcr.setCWE(cwe);
75-
// tcr.setNumber(testNumber(record.get(TEST_NAME), testCaseName));
7681

7782
if (TestCaseResult.UNMAPPED_CATEGORY.equals(tcr.getCategory())) {
7883
System.out.println(
@@ -102,6 +107,20 @@ public static TestSuiteResults parse(ResultFile resultFile) throws IOException {
102107
}
103108

104109
tr.put(tcr);
110+
111+
// Add this test case to the testcaseCountsPerCWE Map, if CategoryGroups enabled
112+
if (CategoryGroups.isCategoryGroupsEnabled()) {
113+
CountsPerCWE cweCounts = testcaseCountsPerCWE.get(cweNumber);
114+
if (cweCounts == null) {
115+
cweCounts = new CountsPerCWE();
116+
if (truePositive) cweCounts.truePositiveCount = 1;
117+
else cweCounts.falsePositiveCount = 1;
118+
testcaseCountsPerCWE.put(cweNumber, cweCounts);
119+
} else {
120+
if (truePositive) cweCounts.truePositiveCount++;
121+
else cweCounts.falsePositiveCount++;
122+
}
123+
}
105124
}
106125
}
107126

@@ -182,4 +201,16 @@ private static void setExpectedResultsMetadata(
182201
private static boolean isExtendedResultsFile(CSVParser parser) {
183202
return parser.getHeaderNames().contains(SOURCE);
184203
}
204+
205+
/**
206+
* Gets the number of True Positive and/or False Positive test cases in this test suite for the
207+
* specified CWE number.
208+
*
209+
* @param cweNumber The CWE number to look for
210+
* @return The Counts for that CWE, or null if there are no test cases for this CWE in this test
211+
* suite
212+
*/
213+
public static CountsPerCWE getTestcaseCountsForCWE(int cwe) {
214+
return testcaseCountsPerCWE.get(Integer.toString(cwe));
215+
}
185216
}

‎plugin/src/main/resources/scorecard/vulntemplate.html‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ <h2>Detailed Results Per Tool for ${vulnerability}</h2>
6262
${table}
6363

6464
<p />
65+
${CategoryGroupDetailsTitle} ${CategoryGroupDetailsTable}
6566
<p />
6667

6768
<h2>Key</h2>

0 commit comments

Comments
 (0)