Merge pull request vllm-project#676 from vllm-project/test/fail-on-zero-accuracy

[CI/Build] Fail e2e tests when accuracy is 0%

Author: wangchen615

e2e/testcases/domain_classify.go

@@ -92,6 +92,11 @@ func testDomainClassify(ctx context.Context, client *kubernetes.Clientset, opts
 			correctTests, totalTests, accuracy)
 	}
 
+	// Return error if accuracy is 0%
+	if correctTests == 0 {
+		return fmt.Errorf("domain classification test failed: 0%% accuracy (0/%d correct)", totalTests)
+	}
+
 	return nil
 }
 

e2e/testcases/jailbreak_detection.go

@@ -103,6 +103,11 @@ func testJailbreakDetection(ctx context.Context, client *kubernetes.Clientset, o
 			correctTests, totalTests, detectionRate)
 	}
 
+	// Return error if detection rate is 0%
+	if correctTests == 0 {
+		return fmt.Errorf("jailbreak detection test failed: 0%% accuracy (0/%d correct)", totalTests)
+	}
+
 	return nil
 }
 

e2e/testcases/pii_detection.go

@@ -103,6 +103,11 @@ func testPIIDetection(ctx context.Context, client *kubernetes.Clientset, opts pk
 			correctTests, totalTests, detectionRate)
 	}
 
+	// Return error if detection rate is 0%
+	if correctTests == 0 {
+		return fmt.Errorf("PII detection test failed: 0%% accuracy (0/%d correct)", totalTests)
+	}
+
 	return nil
 }