Use smaller batch size in run-tests.php when appropriate

- When valgrind is used, communication overhead is relatively small,
  so just use a batch size of 1.
- If this is running a small enough number of tests,
  reduce the batch size to give batches to more workers.
  (Previously, if there were 90 tests and -j8,
  only 3 of 8 workers would get a batch of size 32 or less.
  After this change, the batch size is 12 or less)

Closes GH-5098

Author: TysonAndre

run-tests.php

@@ -1341,7 +1341,7 @@ function run_all_tests($test_files, $env, $redir_tested = null)
 
 /** The heart of parallel testing. */
 function run_all_tests_parallel($test_files, $env, $redir_tested) {
-	global $workers, $test_idx, $test_cnt, $test_results, $failed_tests_file, $result_tests_file, $PHP_FAILED_TESTS, $shuffle, $SHOW_ONLY_GROUPS;
+	global $workers, $test_idx, $test_cnt, $test_results, $failed_tests_file, $result_tests_file, $PHP_FAILED_TESTS, $shuffle, $SHOW_ONLY_GROUPS, $valgrind;
 
 	// The PHP binary running run-tests.php, and run-tests.php itself
 	// This PHP executable is *not* necessarily the same as the tested version
@@ -1415,6 +1415,7 @@ function run_all_tests_parallel($test_files, $env, $redir_tested) {
 	}
 	$sockPort = substr($sockName, $portPos + 1);
 	$sockUri = "tcp://$sockHost:$sockPort";
+	$totalFileCount = count($test_files);
 
 	for ($i = 1; $i <= $workers; $i++) {
 		$proc = proc_open(
@@ -1548,8 +1549,14 @@ function run_all_tests_parallel($test_files, $env, $redir_tested) {
 								$sequentialTests = [];
 							}
 							// Batch multiple tests to reduce communication overhead.
+							// - When valgrind is used, communication overhead is relatively small,
+							//   so just use a batch size of 1.
+							// - If this is running a small enough number of tests,
+							//   reduce the batch size to give batches to more workers.
 							$files = [];
-							$batchSize = $shuffle ? 4 : 32;
+							$maxBatchSize = $valgrind ? 1 : ($shuffle ? 4 : 32);
+							$averageFilesPerWorker = max(1, (int)ceil($totalFileCount / count($workerProcs)));
+							$batchSize = min($maxBatchSize, $averageFilesPerWorker);
 							while (count($files) <= $batchSize && $file = array_pop($test_files)) {
 								foreach ($fileConflictsWith[$file] as $conflictKey) {
 									if (isset($activeConflicts[$conflictKey])) {