Skip to content

Commit 6bf2afc

Browse files
author
Thinh Nguyen
authored
improve populate returns to be consistent with the input kwargs
1 parent 0abd3c0 commit 6bf2afc

File tree

1 file changed

+35
-35
lines changed

1 file changed

+35
-35
lines changed

datajoint/autopopulate.py

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -214,49 +214,49 @@ def handler(signum, frame):
214214

215215
keys = keys[:max_calls]
216216
nkeys = len(keys)
217-
if not nkeys:
218-
return
219-
220-
processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _)
221-
217+
222218
error_list = []
223219
success_list = []
224-
populate_kwargs = dict(
225-
suppress_errors=suppress_errors,
226-
return_exception_objects=return_exception_objects,
227-
make_kwargs=make_kwargs,
228-
)
229220

230-
if processes == 1:
231-
for key in (
232-
tqdm(keys, desc=self.__class__.__name__) if display_progress else keys
233-
):
234-
status = self._populate1(key, jobs, **populate_kwargs)
235-
if status is not None:
236-
if isinstance(status, tuple):
237-
error_list.append(status)
238-
elif status:
239-
success_list.append(1)
240-
else:
241-
# spawn multiple processes
242-
self.connection.close() # disconnect parent process from MySQL server
243-
del self.connection._conn.ctx # SSLContext is not pickleable
244-
with mp.Pool(
245-
processes, _initialize_populate, (self, jobs, populate_kwargs)
246-
) as pool, (
247-
tqdm(desc="Processes: ", total=nkeys)
248-
if display_progress
249-
else contextlib.nullcontext()
250-
) as progress_bar:
251-
for status in pool.imap(_call_populate1, keys, chunksize=1):
221+
if nkeys:
222+
processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _)
223+
224+
populate_kwargs = dict(
225+
suppress_errors=suppress_errors,
226+
return_exception_objects=return_exception_objects,
227+
make_kwargs=make_kwargs,
228+
)
229+
230+
if processes == 1:
231+
for key in (
232+
tqdm(keys, desc=self.__class__.__name__) if display_progress else keys
233+
):
234+
status = self._populate1(key, jobs, **populate_kwargs)
252235
if status is not None:
253236
if isinstance(status, tuple):
254237
error_list.append(status)
255238
elif status:
256239
success_list.append(1)
257-
if display_progress:
258-
progress_bar.update()
259-
self.connection.connect() # reconnect parent process to MySQL server
240+
else:
241+
# spawn multiple processes
242+
self.connection.close() # disconnect parent process from MySQL server
243+
del self.connection._conn.ctx # SSLContext is not pickleable
244+
with mp.Pool(
245+
processes, _initialize_populate, (self, jobs, populate_kwargs)
246+
) as pool, (
247+
tqdm(desc="Processes: ", total=nkeys)
248+
if display_progress
249+
else contextlib.nullcontext()
250+
) as progress_bar:
251+
for status in pool.imap(_call_populate1, keys, chunksize=1):
252+
if status is not None:
253+
if isinstance(status, tuple):
254+
error_list.append(status)
255+
elif status:
256+
success_list.append(1)
257+
if display_progress:
258+
progress_bar.update()
259+
self.connection.connect() # reconnect parent process to MySQL server
260260

261261
# restore original signal handler:
262262
if reserve_jobs:

0 commit comments

Comments
 (0)