Skip to content

Commit 4e35a13

Browse files
committed
Relocate context definition and add back comments
1 parent fa60f14 commit 4e35a13

File tree

1 file changed

+4
-1
lines changed

1 file changed

+4
-1
lines changed

src/mdio/segy/blocked_io.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,17 @@ def to_zarr(
6060
# Determine number of workers
6161
num_cpus_env = int(os.getenv("MDIO__IMPORT__CPU_COUNT", default_cpus))
6262
num_workers = min(num_chunks, num_cpus_env)
63-
context = mp.get_context("spawn")
6463

6564
# Chunksize here is for multiprocessing, not Zarr chunksize.
6665
pool_chunksize, extra = divmod(num_chunks, num_workers * 4)
6766
pool_chunksize += 1 if extra else 0
6867

6968
tqdm_kw = {"unit": "block", "dynamic_ncols": True}
7069

70+
# For Unix async writes with s3fs/fsspec & multiprocessing, use 'spawn' instead of default
71+
# 'fork' to avoid deadlocks on cloud stores. Slower but necessary. Default on Windows
72+
context = mp.get_context("spawn")
73+
7174
# Launch multiprocessing pool
7275
with ProcessPoolExecutor(max_workers=num_workers, mp_context=context) as executor:
7376
lazy_work = executor.map(

0 commit comments

Comments
 (0)