Merge pull request ClickHouse#79303 from ClickHouse/blacklist-update

Stateless tests: mark blacklisted tests as failed when they succeed

Author: fm4v

ci/jobs/scripts/functional_tests_results.py

@@ -26,10 +26,6 @@
 #         out = csv.writer(f, delimiter="\t")
 #         out.writerow(status)
 
-BROKEN_TESTS_ANALYZER_TECH_DEBT = [
-    "01624_soft_constraints",
-]
-
 
 class FTResultsProcessor:
     @dataclasses.dataclass
@@ -50,7 +46,6 @@ def __init__(self, wd):
         self.tests_output_file = f"{wd}/test_result.txt"
         # self.test_results_parsed_file = f"{wd}/test_result.tsv"
         # self.status_file = f"{wd}/check_status.tsv"
-        self.broken_tests = BROKEN_TESTS_ANALYZER_TECH_DEBT
 
     def _process_test_output(self):
         total = 0
@@ -97,41 +92,20 @@ def _process_test_output(self):
 
                     total += 1
                     if TIMEOUT_SIGN in line:
-                        if test_name in self.broken_tests:
-                            success += 1
-                            test_results.append((test_name, "BROKEN", test_time, []))
-                        else:
-                            failed += 1
-                            test_results.append((test_name, "Timeout", test_time, []))
+                        failed += 1
+                        test_results.append((test_name, "Timeout", test_time, []))
                     elif FAIL_SIGN in line:
-                        if test_name in self.broken_tests:
-                            success += 1
-                            test_results.append((test_name, "BROKEN", test_time, []))
-                        else:
-                            failed += 1
-                            test_results.append((test_name, "FAIL", test_time, []))
+                        failed += 1
+                        test_results.append((test_name, "FAIL", test_time, []))
                     elif UNKNOWN_SIGN in line:
                         unknown += 1
                         test_results.append((test_name, "FAIL", test_time, []))
                     elif SKIPPED_SIGN in line:
                         skipped += 1
                         test_results.append((test_name, "SKIPPED", test_time, []))
                     else:
-                        if OK_SIGN in line and test_name in self.broken_tests:
-                            skipped += 1
-                            test_results.append(
-                                (
-                                    test_name,
-                                    "NOT_FAILED",
-                                    test_time,
-                                    [
-                                        "This test passed. Update analyzer_tech_debt.txt.\n"
-                                    ],
-                                )
-                            )
-                        else:
-                            success += int(OK_SIGN in line)
-                            test_results.append((test_name, "OK", test_time, []))
+                        success += int(OK_SIGN in line)
+                        test_results.append((test_name, "OK", test_time, []))
                     test_end = False
                 elif (
                     len(test_results) > 0

tests/clickhouse-test

@@ -875,6 +875,7 @@ class TestStatus(enum.Enum):
     UNKNOWN = "UNKNOWN"
     OK = "OK"
     SKIPPED = "SKIPPED"
+    NOT_FAILED = "NOT_FAILED"
 
 
 class FailureReason(enum.Enum):
@@ -917,6 +918,10 @@ class FailureReason(enum.Enum):
     # UNKNOWN reasons
     NO_REFERENCE = "no reference file"
     INTERNAL_ERROR = "Test internal error: "
+    NOT_FAILED = (
+        "The test succeeded, but it is listed in parallel_replicas_blacklist.txt or async_insert_blacklist.txt. "
+        "Please remove it from the list"
+    )
 
 
 def threshold_generator(always_on_prob, always_off_prob, min_val, max_val):
@@ -1543,7 +1548,7 @@ class TestCase:
 
         return None
 
-    def process_result_impl(self, proc, total_time: float):
+    def process_result_impl(self, proc, total_time: float) -> TestResult:
         kill_output = ""
         if proc:
             if proc.returncode is None:
@@ -2050,6 +2055,13 @@ class TestCase:
             if result.status == TestStatus.FAIL:
                 result.description = self.add_info_about_settings(result.description)
 
+            if self.name in suite.blacklist_check:
+                if result.status == TestStatus.OK:
+                    result.status = TestStatus.NOT_FAILED
+                    result.reason = FailureReason.NOT_FAILED
+                if result.status == TestStatus.FAIL:
+                    result.status = TestStatus.SKIPPED
+
             self._cleanup(result.status == TestStatus.OK)
 
             return result
@@ -2328,6 +2340,10 @@ class TestSuite:
         self.skip_list: List[str] = []
         self.cloud_skip_list: List[str] = []
         self.private_skip_list: List[str] = []
+        # List of tests included in a blacklist.
+        # They will be run, and if they succeed, the status will be changed to FAIL to notify
+        # that the changes in PR fixes this test.
+        self.blacklist_check: List[str] = []
 
         if args.run_by_hash_num is not None and args.run_by_hash_total is not None:
             if args.run_by_hash_num > args.run_by_hash_total:
@@ -2523,12 +2539,18 @@ def run_tests_array(
         + colored(" SKIPPED ", args, "cyan", attrs=["bold"])
         + CL_SQUARE_BRACKET
     )
+    MSG_NOT_FAILED = (
+        OP_SQUARE_BRACKET
+        + colored(" NOT_FAILED ", args, "red", attrs=["bold"])
+        + CL_SQUARE_BRACKET
+    )
 
     MESSAGES = {
         TestStatus.FAIL: MSG_FAIL,
         TestStatus.UNKNOWN: MSG_UNKNOWN,
         TestStatus.OK: MSG_OK,
         TestStatus.SKIPPED: MSG_SKIPPED,
+        TestStatus.NOT_FAILED: MSG_NOT_FAILED,
     }
 
     passed_total = 0
@@ -3294,12 +3316,13 @@ def main(args):
     cloud_skip_list = try_get_skip_list(base_dir, "../queries-no-cloud-tests.txt")
     private_skip_list = try_get_skip_list(base_dir, "../queries-no-private-tests.txt")
     skip_list = []
+    blacklist_check = []
+
+    blacklist_check.extend(try_get_skip_list(base_dir, "../analyzer_tech_debt.txt"))
 
     if args.no_parallel_replicas is True:
-        skip_list_parallel_replicas = try_get_skip_list(
-            base_dir, "../parallel_replicas_blacklist.txt"
-        )
-        skip_list.extend(skip_list_parallel_replicas)
+        blacklist = try_get_skip_list(base_dir, "../parallel_replicas_blacklist.txt")
+        blacklist_check.extend(blacklist)
 
     for suite in sorted(os.listdir(base_dir), key=suite_key_func):
         if server_died.is_set():
@@ -3312,6 +3335,7 @@ def main(args):
         test_suite.skip_list = skip_list
         test_suite.cloud_skip_list = cloud_skip_list
         test_suite.private_skip_list = private_skip_list
+        test_suite.blacklist_check = blacklist_check
         total_tests_run += do_run_tests(
             args.jobs, test_suite, args, exit_code, restarted_tests, server_died
         )

tests/parallel_replicas_blacklist.txt

@@ -513,7 +513,6 @@
 
 00955_test_final_mark_use
 01624_soft_constraints
-02968_file_log_multiple_read
 03031_minmax_index_for_pointinpolygon
 
     ORDER BY ALL is missing for multiple queries

tests/queries/0_stateless/02968_file_log_multiple_read.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-parallel-replicas
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh