Stateless tests: mark blacklisted tests as failed when they succeed

Author: fm4v

ci/jobs/scripts/functional_tests_results.py

@@ -26,10 +26,6 @@
 #         out = csv.writer(f, delimiter="\t")
 #         out.writerow(status)
 
-BROKEN_TESTS_ANALYZER_TECH_DEBT = [
-    "01624_soft_constraints",
-]
-
 
 class FTResultsProcessor:
     @dataclasses.dataclass
@@ -50,7 +46,6 @@ def __init__(self, wd):
         self.tests_output_file = f"{wd}/test_result.txt"
         # self.test_results_parsed_file = f"{wd}/test_result.tsv"
         # self.status_file = f"{wd}/check_status.tsv"
-        self.broken_tests = BROKEN_TESTS_ANALYZER_TECH_DEBT
 
     def _process_test_output(self):
         total = 0
@@ -97,41 +92,20 @@ def _process_test_output(self):
 
                     total += 1
                     if TIMEOUT_SIGN in line:
-                        if test_name in self.broken_tests:
-                            success += 1
-                            test_results.append((test_name, "BROKEN", test_time, []))
-                        else:
-                            failed += 1
-                            test_results.append((test_name, "Timeout", test_time, []))
+                        failed += 1
+                        test_results.append((test_name, "Timeout", test_time, []))
                     elif FAIL_SIGN in line:
-                        if test_name in self.broken_tests:
-                            success += 1
-                            test_results.append((test_name, "BROKEN", test_time, []))
-                        else:
-                            failed += 1
-                            test_results.append((test_name, "FAIL", test_time, []))
+                        failed += 1
+                        test_results.append((test_name, "FAIL", test_time, []))
                     elif UNKNOWN_SIGN in line:
                         unknown += 1
                         test_results.append((test_name, "FAIL", test_time, []))
                     elif SKIPPED_SIGN in line:
                         skipped += 1
                         test_results.append((test_name, "SKIPPED", test_time, []))
                     else:
-                        if OK_SIGN in line and test_name in self.broken_tests:
-                            skipped += 1
-                            test_results.append(
-                                (
-                                    test_name,
-                                    "NOT_FAILED",
-                                    test_time,
-                                    [
-                                        "This test passed. Update analyzer_tech_debt.txt.\n"
-                                    ],
-                                )
-                            )
-                        else:
-                            success += int(OK_SIGN in line)
-                            test_results.append((test_name, "OK", test_time, []))
+                        success += int(OK_SIGN in line)
+                        test_results.append((test_name, "OK", test_time, []))
                     test_end = False
                 elif (
                     len(test_results) > 0

tests/clickhouse-test

@@ -3318,6 +3318,8 @@ def main(args):
     skip_list = []
     blacklist_check = []
 
+    blacklist_check.extend(try_get_skip_list(base_dir, "../analyzer_tech_debt.txt"))
+
     if args.no_parallel_replicas is True:
         blacklist = try_get_skip_list(base_dir, "../parallel_replicas_blacklist.txt")
         blacklist_check.extend(blacklist)