Support inline tag exclusions

Author: msimacek

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -448,19 +448,20 @@ def generate_tags(self, append=False):
             by_file[result.test_id.test_file].append(result)
         for test_file, results in by_file.items():
             test_file = configure_test_file(test_file)
-            tags = update_tags(
-                current=read_tags(test_file),
-                results=results,
+            update_tags(
+                test_file,
+                results,
                 tag_platform=CURRENT_PLATFORM,
                 untag_failed=(not append),
                 untag_skipped=(not append),
                 untag_missing=(not append),
             )
-            write_tags(test_file, tags)
 
 
-def update_tags(current: typing.Iterable['Tag'], results: typing.Iterable[TestResult], tag_platform: str,
-                untag_failed=False, untag_skipped=False, untag_missing=False) -> set['Tag']:
+def update_tags(test_file: 'TestFile', results: list[TestResult], tag_platform: str,
+                untag_failed=False, untag_skipped=False, untag_missing=False):
+    current = read_tags(test_file, allow_exclusions=True)
+    exclusions, current = partition_list(current, lambda t: isinstance(t, TagExclusion))
     status_by_id = {r.test_id.normalized(): r.status for r in results}
     tag_by_id = {}
     for tag in current:
@@ -483,7 +484,11 @@ def update_tags(current: typing.Iterable['Tag'], results: typing.Iterable[TestRe
         elif status == TestStatus.SUCCESS:
             tag_by_id[test_id] = Tag.for_key(test_id, tag_platform)
 
-    return set(tag_by_id.values())
+    for exclusion in exclusions:
+        tag_by_id.pop(exclusion.test_id, None)
+
+    tags = set(tag_by_id.values()) | set(exclusions)
+    write_tags(test_file, tags)
 
 
 def write_tags(test_file: 'TestFile', tags: typing.Iterable['Tag']):
@@ -493,6 +498,7 @@ def write_tags(test_file: 'TestFile', tags: typing.Iterable['Tag']):
         return
     if not tags:
         tag_file.unlink(missing_ok=True)
+        return
     with open(tag_file, 'w') as f:
         for tag in sorted(tags, key=lambda t: t.test_id.test_name):
             f.write(f'{tag}\n')
@@ -1149,23 +1155,53 @@ def __str__(self):
         return f'{self.test_id.test_name} @ {",".join(sorted(self.keys))}'
 
 
-def read_tags(test_file: TestFile) -> list[Tag]:
+@dataclass(frozen=True)
+class TagExclusion(Tag):
+    comment: str | None
+
+    def __str__(self):
+        s = f'!{self.test_id.test_name}'
+        if self.keys:
+            s += f' @ {",".join(sorted(self.keys))}'
+        if self.comment:
+            s = f'{self.comment}{s}'
+        return s
+
+
+def read_tags(test_file: TestFile, allow_exclusions=False) -> list[Tag]:
     # To make them easily comparable
     test_path = test_file.path.resolve()
     tag_file = test_file.get_tag_file()
     tags = []
     if tag_file.exists():
         with open(tag_file) as f:
+            comment = None
             for line in f:
+                if line.startswith('#'):
+                    if comment:
+                        comment += line
+                    else:
+                        comment = line
+                    continue
                 test, _, keys = line.partition('@')
                 test = test.strip()
                 keys = keys.strip()
-                if not keys:
-                    log(f'WARNING: invalid tag {test}: missing platform keys')
-                tags.append(Tag(
-                    TestId(test_path, test),
-                    frozenset(keys.split(',')),
-                ))
+                if test.startswith('!'):
+                    if allow_exclusions:
+                        test = test.removeprefix('!')
+                        tags.append(TagExclusion(
+                            TestId(test_path, test),
+                            frozenset(keys.split(',')) if keys else frozenset(),
+                            comment,
+                        ))
+                else:
+                    if not keys:
+                        log(f'WARNING: invalid tag {test}: missing platform keys')
+                    tags.append(Tag(
+                        TestId(test_path, test),
+                        frozenset(keys.split(',')),
+                    ))
+                comment = None
     return tags
 
 
@@ -1227,15 +1263,14 @@ def main_merge_tags(args):
         by_file[result.test_id.test_file].append(result)
     for test_file, results in by_file.items():
         test_file = configure_test_file(test_file)
-        tags = update_tags(
-            current=read_tags(test_file),
-            results=results,
+        update_tags(
+            test_file,
+            results,
             tag_platform=args.platform,
             untag_failed=False,
             untag_skipped=True,
             untag_missing=True,
         )
-        write_tags(test_file, tags)
 
 
 def get_bool_env(name: str):

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_buffer.txt

@@ -16,7 +16,8 @@ test.test_buffer.TestBufferProtocol.test_memoryview_tolist @ darwin-arm64,darwin
 test.test_buffer.TestBufferProtocol.test_ndarray_cmp_contig @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_fortran @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_hash @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_buffer.TestBufferProtocol.test_ndarray_index_getitem_multidim @ darwin-arm64,linux-aarch64,linux-x86_64
+# Transient lists differ error GR-49936
+!test.test_buffer.TestBufferProtocol.test_ndarray_index_getitem_multidim @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_index_invalid @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_index_null_strides @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_index_scalar @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
@@ -25,9 +26,10 @@ test.test_buffer.TestBufferProtocol.test_ndarray_memoryview_from_buffer @ darwin
 test.test_buffer.TestBufferProtocol.test_ndarray_offset @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_random_invalid @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_re_export @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_buffer.TestBufferProtocol.test_ndarray_slice_assign_multidim @ darwin-arm64,linux-aarch64,linux-x86_64
-test.test_buffer.TestBufferProtocol.test_ndarray_slice_multidim @ darwin-arm64,linux-aarch64,linux-x86_64
-test.test_buffer.TestBufferProtocol.test_ndarray_slice_redundant_suboffsets @ darwin-arm64,linux-aarch64,linux-x86_64
+# Transient lists differ error GR-49936
+!test.test_buffer.TestBufferProtocol.test_ndarray_slice_assign_multidim @ darwin-arm64,linux-aarch64,linux-x86_64
+!test.test_buffer.TestBufferProtocol.test_ndarray_slice_multidim @ darwin-arm64,linux-aarch64,linux-x86_64
+!test.test_buffer.TestBufferProtocol.test_ndarray_slice_redundant_suboffsets @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_slice_zero_shape @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_tolist_null_strides @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_buffer.TestBufferProtocol.test_ndarray_zero_shape @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_concurrent_futures.txt

@@ -47,8 +47,9 @@ test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecuto
 test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_max_tasks_per_child_defaults_to_spawn_context @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_max_workers_negative @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecutorTest.test_idle_process_reuse_multiple @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecutorTest.test_idle_process_reuse_one @ darwin-arm64,linux-aarch64,linux-x86_64
-test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecutorTest.test_killed_child @ darwin-arm64,linux-aarch64,linux-x86_64
+# Transiently ends up with 2 processes
+!test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecutorTest.test_idle_process_reuse_one @ darwin-arm64,linux-aarch64,linux-x86_64
+!test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecutorTest.test_killed_child @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecutorTest.test_map @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecutorTest.test_map_chunksize @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecutorTest.test_map_exception @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
@@ -65,12 +66,15 @@ test.test_concurrent_futures.test_process_pool.ProcessPoolSpawnProcessPoolExecut
 test.test_concurrent_futures.test_shutdown.ProcessPoolForkProcessPoolShutdownTest.test_run_after_shutdown @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_cancel_futures @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_context_manager_shutdown @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
+# Transiently times out GR-52666
+!test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_del_shutdown
 test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_hang_gh94440 @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_hang_issue12364 @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_processes_terminate @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_run_after_shutdown @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_shutdown_no_wait @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_submit_after_interpreter_shutdown @ darwin-arm64,linux-aarch64,linux-x86_64
+# Transiently times out GR-52666
+!test.test_concurrent_futures.test_shutdown.ProcessPoolSpawnProcessPoolShutdownTest.test_submit_after_interpreter_shutdown @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ThreadPoolShutdownTest.test_cancel_futures @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ThreadPoolShutdownTest.test_context_manager_shutdown @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_shutdown.ThreadPoolShutdownTest.test_hang_gh94440 @ darwin-arm64,linux-aarch64,linux-x86_64
@@ -83,7 +87,7 @@ test.test_concurrent_futures.test_shutdown.ThreadPoolShutdownTest.test_thread_na
 test.test_concurrent_futures.test_shutdown.ThreadPoolShutdownTest.test_threads_terminate @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_thread_pool.ThreadPoolExecutorTest.test_default_workers @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_thread_pool.ThreadPoolExecutorTest.test_executor_map_current_future_cancel @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_concurrent_futures.test_thread_pool.ThreadPoolExecutorTest.test_idle_thread_reuse @ darwin-arm64,linux-aarch64,linux-x86_64
+!test.test_concurrent_futures.test_thread_pool.ThreadPoolExecutorTest.test_idle_thread_reuse @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_thread_pool.ThreadPoolExecutorTest.test_map @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_thread_pool.ThreadPoolExecutorTest.test_map_exception @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_concurrent_futures.test_thread_pool.ThreadPoolExecutorTest.test_map_submits_without_iteration @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_copy.txt

@@ -23,7 +23,7 @@ test.test_copy.TestCopy.test_copy_set @ darwin-arm64,darwin-x86_64,linux-aarch64
 test.test_copy.TestCopy.test_copy_slots @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_copy_tuple @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_copy_tuple_subclass @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_copy.TestCopy.test_copy_weakkeydict @ darwin-arm64,linux-aarch64,linux-x86_64
+!test.test_copy.TestCopy.test_copy_weakkeydict @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_copy_weakref @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_deepcopy_atomic @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_deepcopy_basic @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
@@ -59,7 +59,7 @@ test.test_copy.TestCopy.test_deepcopy_slots @ darwin-arm64,darwin-x86_64,linux-a
 test.test_copy.TestCopy.test_deepcopy_tuple @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_deepcopy_tuple_of_immutables @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_deepcopy_tuple_subclass @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_copy.TestCopy.test_deepcopy_weakkeydict @ darwin-arm64,linux-aarch64,linux-x86_64
+!test.test_copy.TestCopy.test_deepcopy_weakkeydict @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_deepcopy_weakref @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_exceptions @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_copy.TestCopy.test_getstate_exc @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_cprofile.txt

@@ -5,5 +5,6 @@ test.test_profile.ProfileTest.test_calling_conventions @ darwin-arm64,darwin-x86
 test.test_profile.ProfileTest.test_cprofile @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_profile.ProfileTest.test_output_file_when_changing_directory @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_profile.ProfileTest.test_run @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_profile.ProfileTest.test_run_profile_as_module @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
+# This test times out in the gate even though it succeeds locally and in the retagger. Race condition?
+!test.test_profile.ProfileTest.test_run_profile_as_module @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_profile.ProfileTest.test_runctx @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_ctypes.txt

@@ -190,7 +190,8 @@ ctypes.test.test_prototypes.CharPointersTestCase.test_c_void_p_arg_with_c_wchar_
 ctypes.test.test_prototypes.WCharPointersTestCase.test_POINTER_c_wchar_arg @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 ctypes.test.test_prototypes.WCharPointersTestCase.test_c_wchar_p_arg @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 ctypes.test.test_python_api.PythonAPITestCase.test_PyBytes_FromStringAndSize @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-ctypes.test.test_python_api.PythonAPITestCase.test_PyOS_snprintf @ linux-aarch64,linux-x86_64
+# Transient, at least on M1
+!ctypes.test.test_python_api.PythonAPITestCase.test_PyOS_snprintf @ linux-aarch64,linux-x86_64
 ctypes.test.test_returnfuncptrs.ReturnFuncPtrTestCase.test_from_dll @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 ctypes.test.test_returnfuncptrs.ReturnFuncPtrTestCase.test_from_dll_refcount @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 ctypes.test.test_simplesubclasses.Test.test_compare @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_exceptions.txt

@@ -6,7 +6,8 @@ test.test_exceptions.AttributeErrorTests.test_getattr_suggestions_for_same_name
 test.test_exceptions.ExceptionTests.testAttributes @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.testChainingAttrs @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.testChainingDescriptors @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_exceptions.ExceptionTests.testInfiniteRecursion @ darwin-arm64,linux-aarch64,linux-x86_64
+# Disabled due to transient stack overflow that fails to get caught and crashes the VM
+!test.test_exceptions.ExceptionTests.testInfiniteRecursion @ darwin-arm64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.testInvalidAttrs @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.testInvalidTraceback @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.testKeywordArgs @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
@@ -17,7 +18,8 @@ test.test_exceptions.ExceptionTests.testSyntaxErrorMissingParens @ darwin-arm64,
 test.test_exceptions.ExceptionTests.testWithTraceback @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.test_WindowsError @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.test_assert_shadowing @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
-test.test_exceptions.ExceptionTests.test_badisinstance @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
+# Disabled due to transient stack overflow that fails to get caught and crashes the VM
+!test.test_exceptions.ExceptionTests.test_badisinstance @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.test_context_of_exception_in_else_and_finally @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.test_context_of_exception_in_except_and_finally @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_exceptions.ExceptionTests.test_context_of_exception_in_try_and_finally @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_faulthandler.txt

@@ -1,4 +1,7 @@
 test.test_faulthandler.FaultHandlerTests.test_cancel_later_without_dump_traceback_later @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_faulthandler.FaultHandlerTests.test_disable @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
 test.test_faulthandler.FaultHandlerTests.test_is_enabled @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64
+# Disabled since signaling isn't stable during parallel tests
+!test.test_faulthandler.FaultHandlerTests.test_sigbus
+!test.test_faulthandler.FaultHandlerTests.test_sigill
 test.test_faulthandler.FaultHandlerTests.test_sys_xoptions @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64