Fixes to the fix verification feature to make it work for multiple testcases.

Author: dandersonappsecai

library/src/main/java/org/owasp/benchmarkutils/entities/VerifyFixOutput.java

@@ -0,0 +1,96 @@
+/**
+ * OWASP Benchmark Project
+ *
+ * <p>This file is part of the Open Web Application Security Project (OWASP) Benchmark Project For
+ * details, please see <a
+ * href="https://owasp.org/www-project-benchmark/">https://owasp.org/www-project-benchmark/</a>.
+ *
+ * <p>The OWASP Benchmark is free software: you can redistribute it and/or modify it under the terms
+ * of the GNU General Public License as published by the Free Software Foundation, version 2.
+ *
+ * <p>The OWASP Benchmark is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE. See the GNU General Public License for more details.
+ *
+ * @author David Anderson
+ * @created 2024
+ */
+package org.owasp.benchmarkutils.entities;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+public class VerifyFixOutput {
+    private String testCaseName;
+    private ResponseInfo unfixedSafeResponseInfo;
+    private ResponseInfo unfixedAttackResponseInfo;
+    private ResponseInfo fixedSafeResponseInfo;
+    private ResponseInfo fixedAttackResponseInfo;
+    private boolean wasNotVerifiable;
+    private boolean wasExploited;
+    private boolean wasBroken;
+
+    public String getTestCaseName() {
+        return testCaseName;
+    }
+
+    public void setTestCaseName(String testCaseName) {
+        this.testCaseName = testCaseName;
+    }
+
+    public ResponseInfo getUnfixedSafeResponseInfo() {
+        return unfixedSafeResponseInfo;
+    }
+
+    public void setUnfixedSafeResponseInfo(ResponseInfo unfixedSafeResponseInfo) {
+        this.unfixedSafeResponseInfo = unfixedSafeResponseInfo;
+    }
+
+    public ResponseInfo getUnfixedAttackResponseInfo() {
+        return unfixedAttackResponseInfo;
+    }
+
+    public void setUnfixedAttackResponseInfo(ResponseInfo unfixedAttackResponseInfo) {
+        this.unfixedAttackResponseInfo = unfixedAttackResponseInfo;
+    }
+
+    public ResponseInfo getFixedSafeResponseInfo() {
+        return fixedSafeResponseInfo;
+    }
+
+    public void setFixedSafeResponseInfo(ResponseInfo fixedSafeResponseInfo) {
+        this.fixedSafeResponseInfo = fixedSafeResponseInfo;
+    }
+
+    public ResponseInfo getFixedAttackResponseInfo() {
+        return fixedAttackResponseInfo;
+    }
+
+    public void setFixedAttackResponseInfo(ResponseInfo fixedAttackResponseInfo) {
+        this.fixedAttackResponseInfo = fixedAttackResponseInfo;
+    }
+
+    public boolean isWasNotVerifiable() {
+        return wasNotVerifiable;
+    }
+
+    public void setWasNotVerifiable(boolean wasNotVerifiable) {
+        this.wasNotVerifiable = wasNotVerifiable;
+    }
+
+    public boolean isWasExploited() {
+        return wasExploited;
+    }
+
+    public void setWasExploited(boolean wasExploited) {
+        this.wasExploited = wasExploited;
+    }
+
+    public boolean isWasBroken() {
+        return wasBroken;
+    }
+
+    public void setWasBroken(boolean wasBroken) {
+        this.wasBroken = wasBroken;
+    }
+}

plugin/src/main/java/org/owasp/benchmarkutils/helpers/Utils.java

@@ -61,10 +61,10 @@
 import org.eclipse.persistence.oxm.MediaType;
 import org.owasp.benchmarkutils.entities.ResponseInfo;
 import org.owasp.benchmarkutils.entities.TestSuite;
+import org.owasp.benchmarkutils.entities.VerifyFixOutput;
 import org.owasp.benchmarkutils.tools.TestCaseRequestFileParseException;
 import org.owasp.benchmarkutils.tools.TestCaseVerificationResults;
 import org.owasp.benchmarkutils.tools.TestCaseVerificationResultsCollection;
-import org.owasp.benchmarkutils.tools.VerifyFixOutput;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
 

plugin/src/main/java/org/owasp/benchmarkutils/tools/BenchmarkCrawler.java

@@ -139,7 +139,7 @@ void load() {
             new Categories(categoriesFileStream);
 
             this.testSuite = Utils.parseHttpFile(this.theCrawlerFile);
-            System.out.println("Test suite: " + this.testSuite);
+            // System.out.println("Test suite: " + this.testSuite);
             Collections.sort(
                     this.testSuite.getTestCases(),
                     TestCase.getNameComparator()); // Probably not necessary

plugin/src/main/java/org/owasp/benchmarkutils/tools/BenchmarkCrawlerVerification.java

@@ -56,6 +56,7 @@
 import org.owasp.benchmarkutils.entities.TestCaseSetup;
 import org.owasp.benchmarkutils.entities.TestCaseSetupException;
 import org.owasp.benchmarkutils.entities.TestSuite;
+import org.owasp.benchmarkutils.entities.VerifyFixOutput;
 import org.owasp.benchmarkutils.helpers.Utils;
 import org.xml.sax.SAXException;
 
@@ -115,6 +116,10 @@ public class BenchmarkCrawlerVerification extends BenchmarkCrawler {
     private Map<String, TestCaseVerificationResults> testCaseNameToTestCaseVerificationResultsMap =
             new HashMap<>();
 
+    private List<VerifyFixOutput> exploitedFixedTestcases = new ArrayList<>();
+    private List<VerifyFixOutput> brokenFixedTestcases = new ArrayList<>();
+    private List<VerifyFixOutput> notVerifiableFixedTestcases = new ArrayList<>();
+
     BenchmarkCrawlerVerification() {
         // A default constructor required to support Maven plugin API.
         // The theCrawlerFile has to be instantiated before a crawl can be done.
@@ -133,6 +138,7 @@ protected void crawl(TestSuite testSuite) throws Exception {
             List<TestCaseVerificationResults> results =
                     new ArrayList<TestCaseVerificationResults>();
 
+            Files.createDirectories(Paths.get(getOutputDirectory()));
             final File FILE_NON_DISCRIMINATORY_LOG =
                     new File(getOutputDirectory(), FILENAME_NON_DISCRIMINATORY_LOG);
             final File FILE_ERRORS_LOG = new File(getOutputDirectory(), FILENAME_ERRORS_LOG);
@@ -391,6 +397,7 @@ protected void crawl(TestSuite testSuite) throws Exception {
             System.out.printf("Test case time measurements written to: %s%n", FILE_TIMES_LOG);
 
             RegressionTesting.printCrawlSummary(results);
+            printFixVerificationSummary();
             System.out.println();
             System.out.println(completionMessage);
         }
@@ -399,6 +406,15 @@ protected void crawl(TestSuite testSuite) throws Exception {
         // cleanupSetups(setups);
     }
 
+    private void printFixVerificationSummary() {
+        System.out.println("Fix verification summary:");
+        System.out.println();
+        System.out.println("\tExploited fixed test cases:\t" + exploitedFixedTestcases.size());
+        System.out.println("\tBroken fixed test cases:\t" + brokenFixedTestcases.size());
+        System.out.println(
+                "\tNot verifiable fixed test cases:\t" + notVerifiableFixedTestcases.size());
+    }
+
     /**
      * @param testSuite
      * @throws Exception
@@ -599,7 +615,7 @@ private boolean verifyFix(
             TestCaseVerificationResults beforeFixResults,
             TestCaseVerificationResults afterFixResults) {
 
-        boolean wasNotVerfiable =
+        boolean wasNotVerifiable =
                 afterFixResults.getTestCase().isVulnerability()
                         && afterFixResults.getTestCase().isUnverifiable()
                         && afterFixResults.isPassed();
@@ -612,22 +628,32 @@ private boolean verifyFix(
                         .getResponseToSafeValue()
                         .getResponseString()
                         .equals(afterFixResults.getResponseToSafeValue().getResponseString());
-        if (wasNotVerfiable) {
+
+        VerifyFixOutput verifyFixOutput = new VerifyFixOutput();
+        verifyFixOutput.setTestCaseName(afterFixResults.getTestCase().getName());
+        verifyFixOutput.setUnfixedSafeResponseInfo(beforeFixResults.getResponseToSafeValue());
+        verifyFixOutput.setUnfixedAttackResponseInfo(beforeFixResults.getResponseToAttackValue());
+        verifyFixOutput.setFixedSafeResponseInfo(afterFixResults.getResponseToSafeValue());
+        verifyFixOutput.setFixedAttackResponseInfo(afterFixResults.getResponseToAttackValue());
+        verifyFixOutput.setWasNotVerifiable(wasNotVerifiable);
+        verifyFixOutput.setWasExploited(wasExploited);
+        verifyFixOutput.setWasBroken(wasBroken);
+
+        if (wasNotVerifiable) {
             System.out.println("NOT FIXED: Vulnerability could not be verified");
+            notVerifiableFixedTestcases.add(verifyFixOutput);
         }
         if (wasExploited) {
             System.out.println("NOT FIXED: Vulnerability was exploited");
+            exploitedFixedTestcases.add(verifyFixOutput);
         }
         if (wasBroken) {
             System.out.println("NOT FIXED: Functionality was broken");
+            brokenFixedTestcases.add(verifyFixOutput);
         }
 
         File verifyFixResultFile = new File(getOutputDirectory(), FILENAME_VERIFY_FIX_RESULT);
         try (BufferedWriter writer = new BufferedWriter(new FileWriter(verifyFixResultFile))) {
-            VerifyFixOutput verifyFixOutput = new VerifyFixOutput();
-            verifyFixOutput.setWasNotVerfiable(wasNotVerfiable);
-            verifyFixOutput.setWasExploited(wasExploited);
-            verifyFixOutput.setWasBroken(wasBroken);
             String output = Utils.objectToJson(verifyFixOutput);
             //            System.out.println(output);
             writer.write(output);
@@ -640,7 +666,7 @@ private boolean verifyFix(
             e.printStackTrace();
         }
 
-        return !wasNotVerfiable && !wasExploited && !wasBroken;
+        return !wasNotVerifiable && !wasExploited && !wasBroken;
     }
 
     private boolean verifyFixes(