Skip to content

Commit ccd2c82

Browse files
authored
Simplify thread pool used in spec tests runner (#8143)
We can use `imap_unodered` to receive results on the main thread as they arrive instead of using queues and special thread just for printing. I could measure any performance difference.
1 parent 986e7cb commit ccd2c82

File tree

1 file changed

+15
-40
lines changed

1 file changed

+15
-40
lines changed

check.py

Lines changed: 15 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,10 @@
1919
import subprocess
2020
import sys
2121
import unittest
22+
from multiprocessing.pool import ThreadPool
2223
from collections import OrderedDict
23-
from concurrent.futures import ThreadPoolExecutor
2424
from pathlib import Path
25-
import queue
2625
import io
27-
import threading
28-
from functools import partial
2926

3027
from scripts.test import binaryenjs
3128
from scripts.test import lld
@@ -253,54 +250,32 @@ def run_one_spec_test(wast: Path, stdout=None, stderr=None):
253250
check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', test_name + '.log'), stdout=stdout)
254251

255252

256-
def run_spec_test_with_wrapped_stdout(output_queue, wast: Path):
253+
def run_spec_test_with_wrapped_stdout(wast: Path):
257254
out = io.StringIO()
258255
try:
259-
ret = run_one_spec_test(wast, stdout=out, stderr=out)
256+
run_one_spec_test(wast, stdout=out, stderr=out)
260257
except Exception as e:
258+
# Serialize exceptions into the output string buffer
259+
# so they can be reported on the main thread.
261260
print(e, file=out)
262261
raise
263-
finally:
264-
# If a test fails, it's important to keep its output
265-
output_queue.put(out.getvalue())
266-
return ret
262+
return out.getvalue()
267263

268264

269265
def run_spec_tests():
270266
print('\n[ checking wasm-shell spec testcases... ]\n')
271267

272-
output_queue = queue.Queue()
273-
274-
stop_printer = object()
275-
276-
def printer():
277-
while True:
278-
string = output_queue.get()
279-
if string is stop_printer:
280-
break
281-
282-
print(string, end="")
283-
284-
printing_thread = threading.Thread(target=printer)
285-
printing_thread.start()
286-
287268
worker_count = os.cpu_count()
288269
print("Running with", worker_count, "workers")
289-
executor = ThreadPoolExecutor(max_workers=worker_count)
290-
try:
291-
results = executor.map(partial(run_spec_test_with_wrapped_stdout, output_queue), map(Path, shared.options.spec_tests))
292-
for _ in results:
293-
# Iterating joins the threads. No return value here.
294-
pass
295-
except KeyboardInterrupt:
296-
# Hard exit to avoid threads continuing to run after Ctrl-C.
297-
# There's no concern of deadlocking during shutdown here.
298-
os._exit(1)
299-
finally:
300-
executor.shutdown(cancel_futures=True)
301-
302-
output_queue.put(stop_printer)
303-
printing_thread.join()
270+
test_paths = [Path(x) for x in shared.options.spec_tests]
271+
with ThreadPool(processes=worker_count) as pool:
272+
try:
273+
for result in pool.imap_unordered(run_spec_test_with_wrapped_stdout, test_paths):
274+
print(result, end="")
275+
except KeyboardInterrupt:
276+
# Hard exit to avoid threads continuing to run after Ctrl-C.
277+
# There's no concern of deadlocking during shutdown here.
278+
os._exit(1)
304279

305280

306281
def run_validator_tests():

0 commit comments

Comments
 (0)