Skip to content

Commit 9eb4eb5

Browse files
davewichersdavewichers
committed
Add new scorecard generators for Checkmarx and Zap. Prepare for update to 1.2beta release.
1 parent 0d1beab commit 9eb4eb5

27 files changed

+662
-188
lines changed

‎runBenchmark.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
call mvn compile verify cargo:run -Pdeploy

‎runBenchmark.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/sh
2+
3+
chmod 755 src/main/resources/insecureCmd.sh
4+
mvn clean compile verify cargo:run -Pdeploy

‎src/config/note.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<?xml version="1.0" encoding="UTF-8"?><note><to>Tove</to><from>Jani</from><heading>Reminder</heading><body>Don't forget me this weekend!</body></note>

‎src/main/java/org/owasp/benchmark/score/BenchmarkScore.java

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@
4141
import java.util.TreeMap;
4242

4343
import org.apache.commons.io.FileUtils;
44-
4544
import org.owasp.benchmark.score.parsers.AppscanReader;
45+
import org.owasp.benchmark.score.parsers.CheckmarxReader;
4646
import org.owasp.benchmark.score.parsers.Counter;
4747
import org.owasp.benchmark.score.parsers.CoverityReader;
4848
import org.owasp.benchmark.score.parsers.FindbugsReader;
@@ -54,6 +54,7 @@
5454
import org.owasp.benchmark.score.parsers.TestCaseResult;
5555
import org.owasp.benchmark.score.parsers.TestResults;
5656
import org.owasp.benchmark.score.parsers.VeracodeReader;
57+
import org.owasp.benchmark.score.parsers.ZapReader;
5758
import org.owasp.benchmark.score.report.Report;
5859
import org.owasp.benchmark.score.report.ScatterScores;
5960
import org.owasp.benchmark.score.report.ScatterVulns;
@@ -97,11 +98,12 @@ public static void main(String[] args) {
9798
} else {
9899
System.out.println("Deleting previously generated scorecard files in: " + scoreCardDir.getAbsolutePath());
99100
FileUtils.cleanDirectory(scoreCardDir);
100-
101-
// now copy the entire /content directory, that was just deleted with everything else
102-
File dest1 = new File(scoreCardDirName + File.separator + "content");
103-
FileUtils.copyDirectory(new File(pathToScorecardResources + "content"), dest1);
104101
}
102+
103+
// now copy the entire /content directory, that either didn't exist, or was just deleted with everything else
104+
File dest1 = new File(scoreCardDirName + File.separator + "content");
105+
FileUtils.copyDirectory(new File(pathToScorecardResources + "content"), dest1);
106+
105107
} catch (IOException e) {
106108
System.out.println("Error dealing with scorecard directory: '" + scoreCardDir.getAbsolutePath() + "' for some reason!");
107109
e.printStackTrace();
@@ -148,9 +150,12 @@ public static void main(String[] args) {
148150
if ( f.isDirectory() ) {
149151
for ( File actual : f.listFiles() ) {
150152
// Don't confuse the expected results file as an actual results file if its in the same directory
153+
154+
//actual
151155
if (!actual.isDirectory() && !expected.getName().equals(actual.getName()))
152-
process( actual, expectedResults, toolResults );
156+
process( actual, expectedResults, toolResults);
153157
}
158+
//expected
154159
} else {
155160
process( f, expectedResults, toolResults );
156161
}
@@ -210,8 +215,9 @@ private static void process(File f, TestResults expectedResults, List<Report> to
210215
OverallResults results = calculateResults( scores );
211216
results.setTime( actualResults.getTime() );
212217

218+
213219
// This generates the report on disk.
214-
Report scoreCard = new Report( actualResults, scores, results, expectedResults.totalResults(), actualResultsFileName );
220+
Report scoreCard = new Report( actualResults, scores, results, expectedResults.totalResults(), actualResultsFileName,actualResults.isCommercial,true);
215221

216222
// Add this report to the list of reports
217223
toolreports.add(scoreCard);
@@ -353,8 +359,7 @@ private static TestResults readActualResults(File actual) throws Exception {
353359

354360
if ( filename.endsWith(".ozasmt" ) ) {
355361
tr = new AppscanReader().parse( actual );
356-
}
357-
362+
}
358363

359364
else if ( filename.endsWith(".json" ) ) {
360365
tr = new CoverityReader().parse( actual );
@@ -387,6 +392,14 @@ else if ( line2.startsWith( "<detailedreport")) {
387392
else if ( line1.startsWith( "<total")) {
388393
tr = new SonarReader().parse( actual );
389394
}
395+
396+
else if ( line1.contains( "<OWASPZAPReport") || line2.contains( "<OWASPZAPReport")) {
397+
tr = new ZapReader().parse( actual );
398+
}
399+
400+
else if ( line2.startsWith( "<CxXMLResults")) {
401+
tr = new CheckmarxReader().parse( actual );
402+
}
390403
}
391404

392405
else if ( filename.endsWith( ".fpr" ) ) {
@@ -505,7 +518,7 @@ private static boolean compare( TestCaseResult exp, List<TestCaseResult> actList
505518

506519

507520
private static TestResults readExpectedResults(File f1) throws Exception {
508-
TestResults tr = new TestResults( "Expected" );
521+
TestResults tr = new TestResults( "Expected" ,true,null);
509522
BufferedReader fr = new BufferedReader( new FileReader( f1 ) );
510523
boolean reading = true;
511524
while ( reading ) {

‎src/main/java/org/owasp/benchmark/score/WriteTime.java

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@
77
import java.io.IOException;
88
import java.io.InputStreamReader;
99
import java.net.HttpURLConnection;
10+
import java.net.URISyntaxException;
1011
import java.net.URL;
1112
import java.util.List;
13+
import java.util.Properties;
1214

1315
import javax.xml.parsers.DocumentBuilder;
1416
import javax.xml.parsers.DocumentBuilderFactory;
@@ -278,17 +280,24 @@ public String getToolTime(String toolName) {
278280
return "";
279281
}
280282

281-
public String getbenchmarkVersion() throws Exception {
282-
String[] results = new String[2];
283-
List<String> lines = Utils.getLinesFromFile(Utils.getFileFromClasspath(
284-
VERSION_FILE, this.getClass().getClassLoader()));
285-
for (String i : lines) {
286-
results = i.split("=");
287-
// System.out.println(results[1]);
288-
return results[1];
283+
/**
284+
* Gets the current version of the Benchmark from the benchmark.properties file.
285+
* @return The version # (as a String). An empty string if its not defined in that file.
286+
* @throws Exception
287+
*/
288+
public String getbenchmarkVersion() { // throws Exception {
289+
Properties benchMprops = new Properties();
290+
try {
291+
File propsFile = new File(this.getClass().getClassLoader().getResource(VERSION_FILE).toURI().getPath());
292+
benchMprops.load(new FileInputStream(propsFile));
293+
String v = benchMprops.getProperty("version");
294+
if (v == null) return "";
295+
return v;
296+
} catch (IOException | URISyntaxException e) {
297+
System.out.println("Can't load version # from properties file.");
298+
e.printStackTrace();
299+
return "";
289300
}
290-
return "";
291-
292301
}
293302

294303
}

‎src/main/java/org/owasp/benchmark/score/parsers/AppscanReader.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public TestResults parse( File f ) throws Exception {
5050
Map<Integer,String> conf = parsePool( root, "FindingDataPool", "id", "conf", "" );
5151
Map<Integer,Set<Integer>> assess = parseAssessments( root );
5252

53-
TestResults tr = new TestResults( "IBM AppScan Source");
53+
TestResults tr = new TestResults( "IBM AppScan Source",true,TestResults.ToolType.SAST);
5454

5555
// <AssessmentRun name="webgoat-benchmark_3 - 5/18/15 12:01AM" version="9.0.1.0">
5656
String version = getAttributeValue( "version", root );
@@ -85,7 +85,8 @@ public TestResults parse( File f ) throws Exception {
8585
TestCaseResult tcr = new TestCaseResult();
8686
tcr.setNumber( tn );
8787
int vid = Integer.parseInt( finding.get( findingid ) );
88-
int confidence = Integer.parseInt( conf.get( findingid ));
88+
String confString = conf.get( findingid );
89+
int confidence = Integer.parseInt( confString );
8990

9091
String vtype = vulns.get( vid );
9192
tcr.setCategory( vtype );
@@ -94,10 +95,10 @@ public TestResults parse( File f ) throws Exception {
9495
tcr.setEvidence( vtype );
9596
tcr.setConfidence( confidence );
9697

97-
// FIXME - include 3's??
98-
// if ( confidence != 3 ) {
98+
// Exclude 3 and above - apparently these are "scan coverage"
99+
if ( confidence < 3 ) {
99100
tr.put(tcr);
100-
// }
101+
}
101102
}
102103
}
103104
return tr;

‎src/main/java/org/owasp/benchmark/score/parsers/CheckmarxReader.java

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/**
2+
* OWASP Benchmark Project
3+
*
4+
* This file is part of the Open Web Application Security Project (OWASP)
5+
* Benchmark Project For details, please see
6+
* <a href="https://www.owasp.org/index.php/Benchmark">https://www.owasp.org/index.php/Benchmark</a>.
7+
*
8+
* The OWASP Benchmark is free software: you can redistribute it and/or modify it under the terms
9+
* of the GNU General Public License as published by the Free Software Foundation, version 2.
10+
*
11+
* The OWASP Benchmark is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
12+
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU General Public License for more details
14+
*
15+
* @author Dave Wichers <a href="https://www.aspectsecurity.com">Aspect Security</a>
16+
* @created 2015
17+
*/
18+
19+
package org.owasp.benchmark.score.parsers;
20+
21+
import java.io.File;
22+
import java.io.FileInputStream;
23+
import java.text.SimpleDateFormat;
24+
import java.util.List;
25+
26+
import javax.xml.parsers.DocumentBuilder;
27+
import javax.xml.parsers.DocumentBuilderFactory;
28+
29+
import org.w3c.dom.Document;
30+
import org.w3c.dom.Node;
31+
import org.w3c.dom.NodeList;
32+
import org.xml.sax.InputSource;
33+
34+
public class CheckmarxReader extends Reader {
35+
36+
public TestResults parse(File f) throws Exception {
37+
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
38+
DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
39+
InputSource is = new InputSource(new FileInputStream(f));
40+
Document doc = docBuilder.parse(is);
41+
42+
TestResults tr = new TestResults( "Checkmarx CxSAST",true, TestResults.ToolType.SAST );
43+
44+
// <CxXMLResults DeepLink="http://CHECKMARX2/CxWebClient/ViewerMain.aspx?scanid=52869&amp;projectid=30265"
45+
// ScanStart="Monday, July 27, 2015 4:50:08 PM" Preset="Default 2014" ScanTime="13h:54m:20s"
46+
// LinesOfCodeScanned="1507279" FilesScanned="21075" ReportCreationTime="Tuesday, July 28, 2015 8:38:30 AM"
47+
// Team="CxServer" CheckmarxVersion="7.1.8 HF2" ScanComments="" ScanType="Full" SourceOrigin="LocalPath">
48+
49+
Node root = doc.getDocumentElement();
50+
String version = getAttributeValue( "CheckmarxVersion", root );
51+
tr.setToolVersion( version );
52+
53+
String time = getAttributeValue("ScanTime", root);
54+
tr.setTime( time );
55+
56+
List<Node> queryList = getNamedChildren( "Query", root );
57+
58+
for ( Node query : queryList ) {
59+
List<Node> resultList = getNamedChildren( "Result", query );
60+
for ( Node result : resultList ) {
61+
TestCaseResult tcr = parseCheckmarxVulnerability(query, result);
62+
if (tcr != null ) {
63+
tr.put(tcr);
64+
}
65+
}
66+
}
67+
return tr;
68+
}
69+
70+
private TestCaseResult parseCheckmarxVulnerability(Node query, Node result) {
71+
TestCaseResult tcr = new TestCaseResult();
72+
// <Query id="594" cweId="89" name="SQL_Injection" group="Java_High_Risk" Severity="High"
73+
// Language="Java" LanguageHash="0188428345217368" LanguageChangeDate="2015-07-14T00:00:00.0000000"
74+
// SeverityIndex="3">
75+
76+
// <Result NodeId="528692318" FileName="/org/owasp/benchmark/testcode/BenchmarkTest00026.java"
77+
// Status="New" Line="50" Column="29" FalsePositive="False" Severity="High" AssignToUser=""
78+
// state="0" Remark=""
79+
// DeepLink="http://CHECKMARX2/CxWebClient/ViewerMain.aspx?scanid=52869&amp;projectid=30265&amp;pathid=2318"
80+
// SeverityIndex="3">
81+
82+
String cwe = getAttributeValue("cweId", query);
83+
if ( cwe != null ) {
84+
tcr.setCWE( translate( Integer.parseInt(cwe ) ) );
85+
} else {
86+
System.out.println( "flaw: " + query );
87+
}
88+
89+
String name = getAttributeValue("name", query);
90+
tcr.setCategory( name );
91+
// filter out dynamic SQL queries because they report SQL injection separately - these are just dynamic SQL
92+
if ( name.equals( "Dynamic_SQL_Queries" ) ) {
93+
return null;
94+
}
95+
96+
tcr.setConfidence( Integer.parseInt( getAttributeValue( "SeverityIndex", result) ) );
97+
98+
tcr.setEvidence( getAttributeValue( "name", query ) );
99+
100+
String testcase = getAttributeValue( "FileName", result );
101+
testcase = testcase.substring( testcase.lastIndexOf('/') +1);
102+
if ( testcase.startsWith( "BenchmarkTest" ) ) {
103+
String testno = testcase.substring( "BenchmarkTest".length(), testcase.length() -5 );
104+
try {
105+
tcr.setNumber( Integer.parseInt( testno ) );
106+
} catch ( NumberFormatException e ) {
107+
e.printStackTrace();
108+
}
109+
return tcr;
110+
}
111+
112+
return null;
113+
}
114+
115+
private int translate(int cwe) {
116+
switch( cwe ) {
117+
case 77 : return 78; // command injection
118+
case 36 : return 22; // path traversal
119+
case 23 : return 22; // path traversal
120+
case 338: return 330; // weak random
121+
}
122+
return cwe;
123+
}
124+
}

‎src/main/java/org/owasp/benchmark/score/parsers/CoverityReader.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public TestResults parse( File f ) throws Exception {
3434

3535
JSONArray arr = obj.getJSONArray("mergedIssues");
3636

37-
TestResults tr = new TestResults( "Coverity Code Advisor" ); // Coverity's tool is called Code Advisor or Code Advisor On Demand
37+
TestResults tr = new TestResults( "Coverity Code Advisor" ,true,TestResults.ToolType.SAST); // Coverity's tool is called Code Advisor or Code Advisor On Demand
3838
// Fixme: See if we can figure this out from some of the files they provide
3939
tr.setTime(f);
4040

‎src/main/java/org/owasp/benchmark/score/parsers/FindbugsReader.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,12 @@ public TestResults parse( File f ) throws Exception {
3838
InputSource is = new InputSource( new FileInputStream(f) );
3939
Document doc = docBuilder.parse(is);
4040

41-
// FIXME: need a way to figure out if this is SecFindBugs
42-
TestResults tr = new TestResults( "FindBugs" );
41+
TestResults tr = new TestResults( "FindBugs" ,false,TestResults.ToolType.SAST);
4342

4443
// If the filename includes an elapsed time in seconds (e.g., TOOLNAME-seconds.xml), set the compute time on the scorecard.
4544
tr.setTime(f);
4645

47-
// <BugCollection timestamp='1434663265000' analysisTimestamp='1434663273732' sequence='0' release='' version='3.0.1���>
46+
// <BugCollection timestamp='1434663265000' analysisTimestamp='1434663273732' sequence='0' release='' version='3.0.1>
4847
Node root = doc.getDocumentElement();
4948
String version = getAttributeValue( "version", root );
5049
tr.setToolVersion( version );
@@ -127,10 +126,12 @@ private static int figureCWE( TestCaseResult tcr, Node cwenode, Node catnode) {
127126
case "SECXXEDOC" : return 611; // XXE - Probably DOM Parser
128127
case "SECSQLIHIB" : return 564; // Hibernate Injection, child of SQL Injection
129128
case "SECXXESAX" : return 611; // XXE - SAX Parser
129+
case "STAIV" : return 329; // static initialization vector for crypto
130130

131131
case "SECSP" : return 00; // servlet parameter - not a vuln
132132
case "SECSH" : return 00; // servlet header -- not a vuln
133133
case "SECSSQ" : return 00; // servlet query - not a vuln
134+
134135

135136
default : System.out.println( "Unknown category: " + cat );
136137
}

‎src/main/java/org/owasp/benchmark/score/parsers/FortifyReader.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public TestResults parse( File f ) throws Exception {
3838
InputSource is = new InputSource( new FileInputStream(f) );
3939
Document doc = docBuilder.parse(is);
4040

41-
TestResults tr = new TestResults( "HP Fortify" );
41+
TestResults tr = new TestResults( "HP Fortify" ,true,TestResults.ToolType.SAST);
4242
// FIXME - parse real number
4343
tr.setTime("3:38:40");
4444

0 commit comments

Comments
 (0)