Skip to content

Commit 74381f6

Browse files
acmorrowacmorrow
committed
Only add worker threads as necesary
1 parent 7e120e8 commit 74381f6

File tree

3 files changed

+65
-49
lines changed

3 files changed

+65
-49
lines changed

‎SCons/Taskmaster/Job.py

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -474,7 +474,7 @@ def __exit__(self, *args):
474474

475475
def __init__(self, taskmaster, num, stack_size) -> None:
476476
self.taskmaster = taskmaster
477-
self.num_workers = num
477+
self.max_workers = num
478478
self.stack_size = stack_size
479479
self.interrupted = InterruptState()
480480
self.workers = []
@@ -484,7 +484,7 @@ def __init__(self, taskmaster, num, stack_size) -> None:
484484
# also protects access to our state that gets updated
485485
# concurrently. The `can_search_cv` is associated with
486486
# this mutex.
487-
self.tm_lock = (threading.Lock if self.num_workers > 1 else NewParallel.FakeLock)()
487+
self.tm_lock = (threading.Lock if self.max_workers > 1 else NewParallel.FakeLock)()
488488

489489
# Guarded under `tm_lock`.
490490
self.jobs = 0
@@ -493,11 +493,11 @@ def __init__(self, taskmaster, num, stack_size) -> None:
493493
# The `can_search_cv` is used to manage a leader /
494494
# follower pattern for access to the taskmaster, and to
495495
# awaken from stalls.
496-
self.can_search_cv = (threading.Condition if self.num_workers > 1 else NewParallel.FakeCondition)(self.tm_lock)
496+
self.can_search_cv = (threading.Condition if self.max_workers > 1 else NewParallel.FakeCondition)(self.tm_lock)
497497

498498
# The queue of tasks that have completed execution. The
499499
# next thread to obtain `tm_lock`` will retire them.
500-
self.results_queue_lock = (threading.Lock if self.num_workers > 1 else NewParallel.FakeLock)()
500+
self.results_queue_lock = (threading.Lock if self.max_workers > 1 else NewParallel.FakeLock)()
501501
self.results_queue = []
502502

503503
if self.taskmaster.trace:
@@ -516,22 +516,26 @@ def trace_message(self, message) -> None:
516516
method_name = sys._getframe(1).f_code.co_name + "():"
517517
thread_id=threading.get_ident()
518518
self.trace.debug('%s.%s [Thread:%s] %s' % (type(self).__name__, method_name, thread_id, message))
519-
# print('%-15s %s' % (method_name, message))
520519

521520
def start(self) -> None:
522-
if self.num_workers == 1:
521+
if self.max_workers == 1:
523522
self._work()
524523
else:
525-
self._start_workers()
526-
for worker in self.workers:
527-
worker.join()
528-
self.workers = []
524+
self._start_worker()
525+
while len(self.workers) > 0:
526+
self.workers[0].join()
527+
self.workers.pop(0)
529528
self.taskmaster.cleanup()
530529

531-
def _start_workers(self) -> None:
530+
def _maybe_start_worker(self) -> None:
531+
if self.max_workers > 1 and len(self.workers) < self.max_workers:
532+
self._start_worker()
533+
534+
def _start_worker(self) -> None:
532535
prev_size = self._adjust_stack_size()
533-
for _ in range(self.num_workers):
534-
self.workers.append(NewParallel.Worker(self))
536+
if self.trace:
537+
self.trace_message("Starting new worker thread")
538+
self.workers.append(NewParallel.Worker(self))
535539
self._restore_stack_size(prev_size)
536540

537541
def _adjust_stack_size(self):
@@ -680,6 +684,11 @@ def _work(self):
680684
self.trace_message("Found task requiring execution")
681685
self.state = NewParallel.State.READY
682686
self.can_search_cv.notify()
687+
# This thread will be busy taking care of
688+
# `execute`ing this task. If we haven't
689+
# reached the limit, spawn a new thread to
690+
# turn the crank and find the next task.
691+
self._maybe_start_worker()
683692

684693
else:
685694
# We failed to find a task, so this thread

‎test/option/fixture/taskmaster_expected_new_parallel.txt

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
Job.NewParallel._start_worker(): [Thread:XXXXX] Starting new worker thread
12
Job.NewParallel._work(): [Thread:XXXXX] Gained exclusive access
23
Job.NewParallel._work(): [Thread:XXXXX] Starting search
34
Job.NewParallel._work(): [Thread:XXXXX] Found 0 completed tasks to process
@@ -86,5 +87,3 @@ Taskmaster: No candidate anymore.
8687
Job.NewParallel._work(): [Thread:XXXXX] Found no task requiring execution, and have no jobs: marking complete
8788
Job.NewParallel._work(): [Thread:XXXXX] Gained exclusive access
8889
Job.NewParallel._work(): [Thread:XXXXX] Completion detected, breaking from main loop
89-
Job.NewParallel._work(): [Thread:XXXXX] Gained exclusive access
90-
Job.NewParallel._work(): [Thread:XXXXX] Completion detected, breaking from main loop

‎test/option/stack-size.py

Lines changed: 42 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -89,29 +89,29 @@
8989
#
9090
# Test without any options
9191
#
92-
test.run(chdir='work1',
92+
test.run(chdir='work1',
9393
arguments = '.',
9494
stdout=expected_stdout,
9595
stderr='')
9696
test.must_exist(['work1', 'f1.out'])
9797
test.must_exist(['work1', 'f2.out'])
9898

99-
test.run(chdir='work1',
99+
test.run(chdir='work1',
100100
arguments = '-c .')
101101
test.must_not_exist(['work1', 'f1.out'])
102102
test.must_not_exist(['work1', 'f2.out'])
103103

104104
#
105105
# Test with -j2
106106
#
107-
test.run(chdir='work1',
107+
test.run(chdir='work1',
108108
arguments = '-j2 .',
109109
stdout=expected_stdout,
110110
stderr='')
111111
test.must_exist(['work1', 'f1.out'])
112112
test.must_exist(['work1', 'f2.out'])
113113

114-
test.run(chdir='work1',
114+
test.run(chdir='work1',
115115
arguments = '-j2 -c .')
116116
test.must_not_exist(['work1', 'f1.out'])
117117
test.must_not_exist(['work1', 'f2.out'])
@@ -120,29 +120,29 @@
120120
#
121121
# Test with --stack-size
122122
#
123-
test.run(chdir='work1',
123+
test.run(chdir='work1',
124124
arguments = '--stack-size=128 .',
125125
stdout=expected_stdout,
126126
stderr='')
127127
test.must_exist(['work1', 'f1.out'])
128128
test.must_exist(['work1', 'f2.out'])
129129

130-
test.run(chdir='work1',
130+
test.run(chdir='work1',
131131
arguments = '--stack-size=128 -c .')
132132
test.must_not_exist(['work1', 'f1.out'])
133133
test.must_not_exist(['work1', 'f2.out'])
134134

135135
#
136136
# Test with SetOption('stack_size', 128)
137137
#
138-
test.run(chdir='work2',
138+
test.run(chdir='work2',
139139
arguments = '.',
140140
stdout=expected_stdout,
141141
stderr='')
142142
test.must_exist(['work2', 'f1.out'])
143143
test.must_exist(['work2', 'f2.out'])
144144

145-
test.run(chdir='work2',
145+
test.run(chdir='work2',
146146
arguments = '--stack-size=128 -c .')
147147
test.must_not_exist(['work2', 'f1.out'])
148148
test.must_not_exist(['work2', 'f2.out'])
@@ -151,100 +151,108 @@
151151
#
152152
# Test with -j2 --stack-size=128
153153
#
154-
test.run(chdir='work1',
154+
test.run(chdir='work1',
155155
arguments = '-j2 --stack-size=128 .',
156156
stdout=expected_stdout,
157157
stderr='')
158158
test.must_exist(['work1', 'f1.out'])
159159
test.must_exist(['work1', 'f2.out'])
160160

161-
test.run(chdir='work1',
161+
test.run(chdir='work1',
162162
arguments = '-j2 --stack-size=128 -c .')
163163
test.must_not_exist(['work1', 'f1.out'])
164164
test.must_not_exist(['work1', 'f2.out'])
165165

166166
#
167167
# Test with -j2 --stack-size=16
168168
#
169-
test.run(chdir='work1',
169+
test.run(chdir='work1',
170170
arguments = '-j2 --stack-size=16 .',
171171
match=TestSCons.match_re,
172172
stdout=re_expected_stdout,
173173
stderr="""
174174
scons: warning: Setting stack size failed:
175175
size not valid: 16384 bytes
176176
File .*
177+
178+
scons: warning: Setting stack size failed:
179+
size not valid: 16384 bytes
180+
File .*
177181
""")
178182
test.must_exist(['work1', 'f1.out'])
179183
test.must_exist(['work1', 'f2.out'])
180184

181-
test.run(chdir='work1',
185+
test.run(chdir='work1',
182186
arguments = '-j2 --stack-size=16 -c .',
183187
match=TestSCons.match_re,
184188
stderr="""
185189
scons: warning: Setting stack size failed:
186190
size not valid: 16384 bytes
187191
File .*
192+
193+
scons: warning: Setting stack size failed:
194+
size not valid: 16384 bytes
195+
File .*
188196
""")
189197
test.must_not_exist(['work1', 'f1.out'])
190198
test.must_not_exist(['work1', 'f2.out'])
191199

192200
#
193201
# Test with -j2 SetOption('stack_size', 128)
194202
#
195-
test.run(chdir='work2',
203+
test.run(chdir='work2',
196204
arguments = '-j2 .',
197205
stdout=expected_stdout,
198206
stderr='')
199207
test.must_exist(['work2', 'f1.out'])
200208
test.must_exist(['work2', 'f2.out'])
201209

202-
test.run(chdir='work2',
210+
test.run(chdir='work2',
203211
arguments = '-j2 -c .')
204212
test.must_not_exist(['work2', 'f1.out'])
205213
test.must_not_exist(['work2', 'f2.out'])
206214

207215
#
208216
# Test with -j2 --stack-size=128 --warn=no-stack-size
209217
#
210-
test.run(chdir='work1',
218+
test.run(chdir='work1',
211219
arguments = '-j2 --stack-size=128 --warn=no-stack-size .',
212220
stdout=expected_stdout,
213221
stderr='')
214222
test.must_exist(['work1', 'f1.out'])
215223
test.must_exist(['work1', 'f2.out'])
216224

217-
test.run(chdir='work1',
225+
test.run(chdir='work1',
218226
arguments = '-j2 --stack-size=128 --warn=no-stack-size -c .')
219227
test.must_not_exist(['work1', 'f1.out'])
220228
test.must_not_exist(['work1', 'f2.out'])
221229

222230
#
223231
# Test with -j2 --stack-size=16 --warn=no-stack-size
224232
#
225-
test.run(chdir='work1',
233+
test.run(chdir='work1',
226234
arguments = '-j2 --stack-size=16 --warn=no-stack-size .',
227235
stdout=expected_stdout,
228236
stderr='')
229237
test.must_exist(['work1', 'f1.out'])
230238
test.must_exist(['work1', 'f2.out'])
231239

232-
test.run(chdir='work1',
240+
test.run(chdir='work1',
233241
arguments = '-j2 --stack-size=16 --warn=no-stack-size -c .')
234242
test.must_not_exist(['work1', 'f1.out'])
235243
test.must_not_exist(['work1', 'f2.out'])
236244

237245
#
238-
# Test with -j2 --warn=no-stack-size SetOption('stack_size', 128)
246+
# Test with -j2 --warn=no-stack-size SetOption('stack_size', 128)
239247
#
240-
test.run(chdir='work2',
248+
test.run(chdir='work2',
241249
arguments = '-j2 --warn=no-stack-size .',
242250
stdout=expected_stdout,
243251
stderr='')
244252
test.must_exist(['work2', 'f1.out'])
245253
test.must_exist(['work2', 'f2.out'])
246254

247-
test.run(chdir='work2',
255+
test.run(chdir='work2',
248256
arguments = '-j2 --warn=no-stack-size -c .')
249257
test.must_not_exist(['work2', 'f1.out'])
250258
test.must_not_exist(['work2', 'f2.out'])
@@ -254,15 +262,15 @@
254262
#
255263
# Test with -j2 --stack-size=128
256264
#
257-
test.run(chdir='work1',
265+
test.run(chdir='work1',
258266
arguments = '-j2 --stack-size=128 .',
259267
match=TestSCons.match_re,
260268
stdout=re_expected_stdout,
261269
stderr=expect_unsupported)
262270
test.must_exist(['work1', 'f1.out'])
263271
test.must_exist(['work1', 'f2.out'])
264272

265-
test.run(chdir='work1',
273+
test.run(chdir='work1',
266274
arguments = '-j2 --stack-size=128 -c .',
267275
match=TestSCons.match_re,
268276
stderr=expect_unsupported)
@@ -272,15 +280,15 @@
272280
#
273281
# Test with -j2 --stack-size=16
274282
#
275-
test.run(chdir='work1',
283+
test.run(chdir='work1',
276284
arguments = '-j2 --stack-size=16 .',
277285
match=TestSCons.match_re,
278286
stdout=re_expected_stdout,
279287
stderr=expect_unsupported)
280288
test.must_exist(['work1', 'f1.out'])
281289
test.must_exist(['work1', 'f2.out'])
282290

283-
test.run(chdir='work1',
291+
test.run(chdir='work1',
284292
arguments = '-j2 --stack-size=16 -c .',
285293
match=TestSCons.match_re,
286294
stderr=expect_unsupported)
@@ -290,15 +298,15 @@
290298
#
291299
# Test with -j2 SetOption('stack_size', 128)
292300
#
293-
test.run(chdir='work2',
301+
test.run(chdir='work2',
294302
arguments = '-j2 .',
295303
match=TestSCons.match_re,
296304
stdout=re_expected_stdout,
297305
stderr=expect_unsupported)
298306
test.must_exist(['work2', 'f1.out'])
299307
test.must_exist(['work2', 'f2.out'])
300308

301-
test.run(chdir='work2',
309+
test.run(chdir='work2',
302310
arguments = '-j2 -c .',
303311
match=TestSCons.match_re,
304312
stderr=expect_unsupported)
@@ -308,44 +316,44 @@
308316
#
309317
# Test with -j2 --stack-size=128 --warn=no-stack-size
310318
#
311-
test.run(chdir='work1',
319+
test.run(chdir='work1',
312320
arguments = '-j2 --stack-size=128 --warn=no-stack-size .',
313321
stdout=expected_stdout,
314322
stderr='')
315323
test.must_exist(['work1', 'f1.out'])
316324
test.must_exist(['work1', 'f2.out'])
317325

318-
test.run(chdir='work1',
326+
test.run(chdir='work1',
319327
arguments = '-j2 --stack-size=128 --warn=no-stack-size -c .')
320328
test.must_not_exist(['work1', 'f1.out'])
321329
test.must_not_exist(['work1', 'f2.out'])
322330

323331
#
324332
# Test with -j2 --stack-size=16 --warn=no-stack-size
325333
#
326-
test.run(chdir='work1',
334+
test.run(chdir='work1',
327335
arguments = '-j2 --stack-size=16 --warn=no-stack-size .',
328336
stdout=expected_stdout,
329337
stderr='')
330338
test.must_exist(['work1', 'f1.out'])
331339
test.must_exist(['work1', 'f2.out'])
332340

333-
test.run(chdir='work1',
341+
test.run(chdir='work1',
334342
arguments = '-j2 --stack-size=16 --warn=no-stack-size -c .')
335343
test.must_not_exist(['work1', 'f1.out'])
336344
test.must_not_exist(['work1', 'f2.out'])
337345

338346
#
339-
# Test with -j2 --warn=no-stack-size SetOption('stack_size', 128)
347+
# Test with -j2 --warn=no-stack-size SetOption('stack_size', 128)
340348
#
341-
test.run(chdir='work2',
349+
test.run(chdir='work2',
342350
arguments = '-j2 --warn=no-stack-size .',
343351
stdout=expected_stdout,
344352
stderr='')
345353
test.must_exist(['work2', 'f1.out'])
346354
test.must_exist(['work2', 'f2.out'])
347355

348-
test.run(chdir='work2',
356+
test.run(chdir='work2',
349357
arguments = '-j2 --warn=no-stack-size -c .')
350358
test.must_not_exist(['work2', 'f1.out'])
351359
test.must_not_exist(['work2', 'f2.out'])

0 commit comments

Comments
 (0)