[GR-46580] Disable transiently failing tests

PullRequest: graalpython/2830

Author: timfel

graalpython/com.oracle.graal.python.test/src/graalpytest.py

@@ -74,13 +74,13 @@
     'test_int',
 ]
 
-
 # A list of tests with minor transient issues, which can be retried once
 TRANSIENTS = [
     'test_multiprocessing_spawn',
     'test_weakref',
     'test_ssl',
     'test_threading',
+    'test_asyncio',
 ]
 
 os = sys.modules.get("posix", sys.modules.get("nt", None))

graalpython/com.oracle.graal.python.test/src/tests/test_fcntl.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -99,6 +99,7 @@ def test_flock_s_and_x(self):
 
     @unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)')
     @unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility')
+    @unittest.skipUnless("graalpython" in os.environ.get("BITBUCKET_REPO_URL", "graalpython"), "Do not run this in auxillary CI jobs, it can be flaky")
     def test_flock_s_and_s(self):
         os.close(os.open(TEST_FILENAME_FULL_PATH, os.O_WRONLY | os.O_CREAT))
         file = os.open(TEST_FILENAME_FULL_PATH, os.O_WRONLY)

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -55,6 +55,18 @@
 RUNNER = os.path.join(os.path.dirname(__file__), "run_cpython_test.py")
 LINE = "=" * 80
 
+# some CI platforms are more unstable then others when it comes to tests
+# involving many files, sockets, or threads. We skip some tests on some
+# platforms
+SKIPLIST = (
+    ("darwin", None, "test_smtplib"),
+    ("darwin", None, "test_imaplib"),
+    (None, "arm", "test_smtplib"),
+    (None, "aarch64", "test_imaplib"),
+    (None, "arm", "test_multiprocessing_spawn"),
+    (None, "aarch64", "test_multiprocessing_spawn"),
+)
+
 
 def grouper(iterable, n):
     args = [iter(iterable)] * n
@@ -169,7 +181,17 @@ class TestTaggedUnittests(unittest.TestCase):
             selection = set(s.strip() for s in selection.split(","))
             working_tests = [x for x in working_tests if x[0] in selection]
 
+    this_os = sys.platform
+    this_machine = os.uname().machine
     for idx, working_test in enumerate(working_tests):
+        if os.environ.get("CI", None) and any(
+            (os is None or os in this_os)
+            and (machine is None or machine in this_machine)
+            and test == working_test[0]
+            for os, machine, test in SKIPLIST
+        ):
+            print("Skipping test from skiplist for this platform:", working_test[0])
+            continue
         fn = make_test_function(working_test)
         fn.__name__ = "%s[%d/%d]" % (fn.__name__, idx + 1, len(working_tests))
         fn.__qualname__ = "%s.%s" % (TestTaggedUnittests.__name__, fn.__name__)