[GR-23344] Make test_tokenize pass.

PullRequest: graalpython/1504

Author: tomasstupka

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -232,6 +232,9 @@ def parse_unittest_output(output):
                     continue
                 print(f"Suite succeeded with {len(passing_tests)} tests")
                 break
+            elif p.returncode == -9:                
+                print(f"\nTimeout (return code -9)\nyou can try to increase the current timeout {tout}s by using --timeout=NNN")
+                break
             else:
                 print(f"Suite failed, retrying with {len(passing_tests)} tests")
 

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_tokenize.txt

@@ -40,6 +40,9 @@
 *graalpython.lib-python.3.test.test_tokenize.TestRoundtrip.test_backslash_continuation
 *graalpython.lib-python.3.test.test_tokenize.TestRoundtrip.test_continuation
 *graalpython.lib-python.3.test.test_tokenize.TestRoundtrip.test_indentation_semantics_retained
+*graalpython.lib-python.3.test.test_tokenize.TestRoundtrip.test_random_files
+*graalpython.lib-python.3.test.test_tokenize.TestRoundtrip.test_roundtrip
+*graalpython.lib-python.3.test.test_tokenize.TestRoundtrip.test_string_concatenation
 *graalpython.lib-python.3.test.test_tokenize.TestTokenize.test_exact_type
 *graalpython.lib-python.3.test.test_tokenize.TestTokenize.test_oneline_defs
 *graalpython.lib-python.3.test.test_tokenize.TestTokenize.test_pathological_trailing_whitespace
@@ -65,6 +68,7 @@
 *graalpython.lib-python.3.test.test_tokenize.TokenizeTest.test_non_ascii_identifiers
 *graalpython.lib-python.3.test.test_tokenize.TokenizeTest.test_selector
 *graalpython.lib-python.3.test.test_tokenize.TokenizeTest.test_shift
+*graalpython.lib-python.3.test.test_tokenize.TokenizeTest.test_string
 *graalpython.lib-python.3.test.test_tokenize.TokenizeTest.test_tabs
 *graalpython.lib-python.3.test.test_tokenize.TokenizeTest.test_unary
 *graalpython.lib-python.3.test.test_tokenize.TokenizeTest.test_underscore_literals

graalpython/lib-python/3/test/test_tokenize.py

@@ -1616,12 +1616,16 @@ def test_random_files(self):
         for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):
             testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
 
-        if not support.is_resource_enabled("cpu"):
-            testfiles = random.sample(testfiles, 10)
-
+        # XXX Truffle change: checking all files takes way too long
+        # if not support.is_resource_enabled("cpu"):
+        #     testfiles = random.sample(testfiles, 10)
+        print("\npicked 10 random files to check, beware possible transient errors!")
+        testfiles = random.sample(testfiles, 10)
         for testfile in testfiles:
-            if support.verbose >= 2:
-                print('tokenize', testfile)
+            # XXX Truffle change: print files if picking them randomly
+            # if support.verbose >= 2:
+            #     print('tokenize', testfile)
+            print('\tchecking', testfile)
             with open(testfile, 'rb') as f:
                 with self.subTest(file=testfile):
                     self.check_roundtrip(f)