feat: Add --positiveOnly and --negativeOnly argument to run only positive or negative scenarios only across all fuzzers

Author: en-milie

src/main/java/com/endava/cats/args/FilterArguments.java

@@ -186,6 +186,22 @@ enum FormatType {
             description = "Path to custom profile configuration file (YAML)")
     private Path customProfileFile;
 
+    @CommandLine.Option(
+            names = {"--negativeOnly"},
+            description = "Run only fuzzers that expect 4XX responses (validation and error testing scenarios). " +
+                    "This includes fuzzers testing required fields, invalid values, authentication, etc. " +
+                    "Excludes happy path fuzzers and linters.")
+    @Getter
+    private boolean only4xxFuzzers;
+
+    @CommandLine.Option(
+            names = {"--positiveOnly"},
+            description = "Run only fuzzers that expect 2XX responses (happy path and valid data scenarios). " +
+                    "This includes happy path fuzzers, valid examples, and default values. " +
+                    "Excludes validation error fuzzers and linters.")
+    @Getter
+    private boolean only2xxFuzzers;
+
     @Setter
     private TotalCountType totalCountType = TotalCountType.FUZZERS;
 

src/main/java/com/endava/cats/fuzzer/executor/FieldsIteratorExecutor.java

@@ -108,6 +108,11 @@ private void executeTestCase(FieldsIteratorExecutorContext context, String fuzze
         testCaseListener.addExpectedResult(context.getLogger(), "Should return [{}]",
                 context.getExpectedResponseCode() != null ? context.getExpectedResponseCode().asString() : "a response that doesn't match" + matchArguments.getMatchString());
 
+        if (!testCaseListener.shouldContinueExecution(context.getLogger(), context.getExpectedResponseCode())) {
+            testCaseListener.skipTest(context.getLogger(), "Test skipped due to response code filtering");
+            return;
+        }
+
         FuzzingResult fuzzingResult = this.getFuzzingResult(context, fuzzedField, strategy);
 
         CatsResponse response = serviceCaller.call(

src/main/java/com/endava/cats/fuzzer/executor/HeadersIteratorExecutor.java

@@ -70,6 +70,11 @@ public void execute(HeadersIteratorExecutorContext context) {
                             testCaseListener.addExpectedResult(context.getLogger(), "Should return [{}]",
                                     expectedResponseCode != null ? expectedResponseCode.asString() : "a response that doesn't match" + matchArguments.getMatchString());
 
+                            if (!testCaseListener.shouldContinueExecution(context.getLogger(), expectedResponseCode)) {
+                                testCaseListener.skipTest(context.getLogger(), "Test skipped due to response code filtering");
+                                return;
+                            }
+
                             ServiceData serviceData = ServiceData.builder()
                                     .relativePath(context.getFuzzingData().getPath())
                                     .contractPath(context.getFuzzingData().getContractPath())

src/main/java/com/endava/cats/fuzzer/executor/SimpleExecutor.java

@@ -56,6 +56,11 @@ public void execute(SimpleExecutorContext context) {
             testCaseListener.addScenario(context.getLogger(), context.getScenario());
             testCaseListener.addExpectedResult(context.getLogger(), "Should return {}" + context.getExpectedResult(), context.getExpectedSpecificResponseCode());
 
+            if (!testCaseListener.shouldContinueExecution(context.getLogger(), context.getExpectedResponseCode())) {
+                testCaseListener.skipTest(context.getLogger(), "Test skipped due to response code filtering");
+                return;
+            }
+
             CatsResponse response = serviceCaller.call(
                     ServiceData.builder()
                             .relativePath(context.getPath())

src/main/java/com/endava/cats/fuzzer/fields/DuplicateKeysFieldsFuzzer.java

@@ -121,6 +121,11 @@ private void runDuplicationTestCase(FuzzingData data, String field, String dupli
         testCaseListener.addExpectedResult(logger,
                 "Service should return a [{}] response", ResponseCodeFamilyPredefined.FOURXX.asString());
 
+        if (!testCaseListener.shouldContinueExecution(logger, ResponseCodeFamilyPredefined.FOURXX)) {
+            testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+            return;
+        }
+
         CatsResponse response = serviceCaller.call(buildServiceData(data, duplicatedPayload));
         testCaseListener.reportResult(logger, data, response, ResponseCodeFamilyPredefined.FOURXX);
     }

src/main/java/com/endava/cats/fuzzer/fields/InsertWhitespacesInFieldNamesFieldFuzzer.java

@@ -59,6 +59,11 @@ private void process(FuzzingData data, String field, String randomWhitespace) {
         testCaseListener.addScenario(logger, "Insert random whitespaces in the field name [{}]", field);
         testCaseListener.addExpectedResult(logger, "Should get a [{}] response code", ResponseCodeFamilyPredefined.FOURXX);
 
+        if (!testCaseListener.shouldContinueExecution(logger, ResponseCodeFamilyPredefined.FOURXX)) {
+            testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+            return;
+        }
+
         String fuzzedJson = JsonUtils.insertCharactersInFieldKey(data.getPayload(), field, randomWhitespace);
 
         CatsResponse response = serviceCaller.call(ServiceData.builder().relativePath(data.getPath()).headers(data.getHeaders())

src/main/java/com/endava/cats/fuzzer/fields/NewFieldsFuzzer.java

@@ -62,6 +62,11 @@ private void process(FuzzingData data) {
         testCaseListener.addScenario(logger, "Add new field inside the request: name [{}], value [{}]. All other details are similar to a happy flow", NEW_FIELD, NEW_FIELD);
         testCaseListener.addExpectedResult(logger, "Should get a [{}] response code", expectedResultCode.asString());
 
+        if (!testCaseListener.shouldContinueExecution(logger, expectedResultCode)) {
+            testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+            return;
+        }
+
         CatsResponse response = serviceCaller.call(ServiceData.builder().relativePath(data.getPath()).headers(data.getHeaders())
                 .payload(fuzzedJson).queryParams(data.getQueryParams()).httpMethod(data.getMethod()).contractPath(data.getContractPath())
                 .contentType(data.getFirstRequestContentType()).pathParamsPayload(data.getPathParamsPayload())

src/main/java/com/endava/cats/fuzzer/fields/RemoveFieldsFuzzer.java

@@ -5,6 +5,7 @@
 import com.endava.cats.args.ProcessingArguments;
 import com.endava.cats.fuzzer.api.Fuzzer;
 import com.endava.cats.http.HttpMethod;
+import com.endava.cats.http.ResponseCodeFamily;
 import com.endava.cats.http.ResponseCodeFamilyPredefined;
 import com.endava.cats.io.ServiceCaller;
 import com.endava.cats.io.ServiceData;
@@ -89,8 +90,14 @@ private void process(FuzzingData data, List<String> required, Set<String> subset
             testCaseListener.addScenario(logger, "Remove the following fields from request: {}", subset);
 
             boolean hasRequiredFieldsRemove = this.hasRequiredFieldsRemove(required, subset);
+            ResponseCodeFamily expectedResponseCode = ResponseCodeFamilyPredefined.getResultCodeBasedOnRequiredFieldsRemoved(hasRequiredFieldsRemove);
             testCaseListener.addExpectedResult(logger, "Should return [{}] response code as required fields [{}] removed", ResponseCodeFamilyPredefined.getExpectedWordingBasedOnRequiredFields(hasRequiredFieldsRemove));
 
+            if (!testCaseListener.shouldContinueExecution(logger, expectedResponseCode)) {
+                testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+                return;
+            }
+
             CatsResponse response = serviceCaller.call(ServiceData.builder().relativePath(data.getPath()).headers(data.getHeaders())
                     .payload(finalJsonPayload).queryParams(data.getQueryParams()).httpMethod(data.getMethod()).contractPath(data.getContractPath())
                     .contentType(data.getFirstRequestContentType()).pathParamsPayload(data.getPathParamsPayload()).build());

src/main/java/com/endava/cats/fuzzer/fields/ZeroWidthCharsInNamesFieldsFuzzer.java

@@ -64,6 +64,11 @@ private void process(FuzzingData data, String fuzzedField, String fuzzValue) {
                 fuzzedField, FuzzingStrategy.formatValue(fuzzValue));
         testCaseListener.addExpectedResult(logger, "Should get a [{}] response code", ResponseCodeFamilyPredefined.FOURXX.asString());
 
+        if (!testCaseListener.shouldContinueExecution(logger, ResponseCodeFamilyPredefined.FOURXX)) {
+            testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+            return;
+        }
+
         CatsResponse response = serviceCaller.call(ServiceData.builder().relativePath(data.getPath()).headers(data.getHeaders())
                 .payload(fuzzedPayload).queryParams(data.getQueryParams()).httpMethod(data.getMethod()).contractPath(data.getContractPath())
                 .contentType(data.getFirstRequestContentType()).pathParamsPayload(data.getPathParamsPayload())

src/main/java/com/endava/cats/fuzzer/fields/base/BaseFieldsFuzzer.java

@@ -110,6 +110,11 @@ protected void process(FuzzingData data, String fuzzedField, FuzzingStrategy fuz
 
             testCaseListener.addExpectedResult(logger, "Should return [{}]", expectedResponseCodeBasedOnConstraints.asString());
 
+            if (!testCaseListener.shouldContinueExecution(logger, expectedResponseCodeBasedOnConstraints)) {
+                testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+                return;
+            }
+
             CatsResponse response = serviceCaller.call(serviceData);
 
             testCaseListener.reportResult(logger, data, response, expectedResponseCodeBasedOnConstraints, this.shouldMatchResponseSchema(data), this.shouldMatchContentType(data));