pythongh-137242: Add a --no-randomize option, and use it in Android CI (python#138649)

Adds a --no-randomize option to the CI runner, so that randomisation can be easily
disabled for --fast-ci and --slow-ci configurations on single-threaded testing platforms
like Android, iOS, and Emscripten.

---------

Co-authored-by: Malcolm Smith <[email protected]>

Author: vstinner

Android/android.py

@@ -737,12 +737,10 @@ def ci(context):
             # Prove the package is self-contained by using it to run the tests.
             shutil.unpack_archive(package_path, temp_dir)
 
-            # Arguments are similar to --fast-ci, but in single-process mode.
+            # Randomization is disabled because order-dependent failures are
+            # much less likely to pass on a rerun in single-process mode.
             launcher_args = ["--managed", "maxVersion", "-v"]
-            test_args = [
-                "--single-process", "--fail-env-changed", "--rerun", "--slowest",
-                "--verbose3", "-u", "all,-cpu", "--timeout=600"
-            ]
+            test_args = ["--fast-ci", "--single-process", "--no-randomize"]
             run(
                 ["./android.py", "test", *launcher_args, "--", *test_args],
                 cwd=temp_dir

Lib/test/libregrtest/cmdline.py

@@ -271,6 +271,9 @@ def _create_parser():
     group = parser.add_argument_group('Selecting tests')
     group.add_argument('-r', '--randomize', action='store_true',
                        help='randomize test execution order.' + more_details)
+    group.add_argument('--no-randomize', dest='no_randomize', action='store_true',
+                       help='do not randomize test execution order, even if '
+                       'it would be implied by another option')
     group.add_argument('--prioritize', metavar='TEST1,TEST2,...',
                        action='append', type=priority_list,
                        help='select these tests first, even if the order is'
@@ -539,6 +542,8 @@ def _parse_args(args, **kwargs):
                     ns.use_resources.append(r)
     if ns.random_seed is not None:
         ns.randomize = True
+    if ns.no_randomize:
+        ns.randomize = False
     if ns.verbose:
         ns.header = True
 

Lib/test/test_regrtest.py

@@ -182,13 +182,33 @@ def test_randomize(self):
             self.assertTrue(regrtest.randomize)
             self.assertIsInstance(regrtest.random_seed, int)
 
+    def test_no_randomize(self):
+        ns = self.parse_args([])
+        self.assertIs(ns.randomize, False)
+
+        ns = self.parse_args(["--randomize"])
+        self.assertIs(ns.randomize, True)
+
+        ns = self.parse_args(["--no-randomize"])
+        self.assertIs(ns.randomize, False)
+
+        ns = self.parse_args(["--randomize", "--no-randomize"])
+        self.assertIs(ns.randomize, False)
+
+        ns = self.parse_args(["--no-randomize", "--randomize"])
+        self.assertIs(ns.randomize, False)
+
     def test_randseed(self):
         ns = self.parse_args(['--randseed', '12345'])
         self.assertEqual(ns.random_seed, 12345)
         self.assertTrue(ns.randomize)
         self.checkError(['--randseed'], 'expected one argument')
         self.checkError(['--randseed', 'foo'], 'invalid int value')
 
+        ns = self.parse_args(['--randseed', '12345', '--no-randomize'])
+        self.assertEqual(ns.random_seed, 12345)
+        self.assertFalse(ns.randomize)
+
     def test_fromfile(self):
         for opt in '-f', '--fromfile':
             with self.subTest(opt=opt):
@@ -428,11 +448,12 @@ def create_regrtest(self, args):
 
         return regrtest
 
-    def check_ci_mode(self, args, use_resources, rerun=True):
+    def check_ci_mode(self, args, use_resources, *, rerun=True, randomize=True):
         regrtest = self.create_regrtest(args)
         self.assertEqual(regrtest.num_workers, -1)
         self.assertEqual(regrtest.want_rerun, rerun)
-        self.assertTrue(regrtest.randomize)
+        self.assertEqual(regrtest.fail_rerun, False)
+        self.assertEqual(regrtest.randomize, randomize)
         self.assertIsInstance(regrtest.random_seed, int)
         self.assertTrue(regrtest.fail_env_changed)
         self.assertTrue(regrtest.print_slowest)
@@ -469,6 +490,15 @@ def test_slow_ci(self):
         regrtest = self.check_ci_mode(args, use_resources)
         self.assertEqual(regrtest.timeout, 20 * 60)
 
+    def test_ci_no_randomize(self):
+        all_resources = set(cmdline.ALL_RESOURCES)
+        self.check_ci_mode(
+            ["--slow-ci", "--no-randomize"], all_resources, randomize=False
+        )
+        self.check_ci_mode(
+            ["--fast-ci", "--no-randomize"], all_resources - {'cpu'}, randomize=False
+        )
+
     def test_dont_add_python_opts(self):
         args = ['--dont-add-python-opts']
         ns = cmdline._parse_args(args)