File tree Expand file tree Collapse file tree 1 file changed +4
-1
lines changed Expand file tree Collapse file tree 1 file changed +4
-1
lines changed Original file line number Diff line number Diff line change @@ -60,14 +60,17 @@ def to_zarr(
6060 # Determine number of workers
6161 num_cpus_env = int (os .getenv ("MDIO__IMPORT__CPU_COUNT" , default_cpus ))
6262 num_workers = min (num_chunks , num_cpus_env )
63- context = mp .get_context ("spawn" )
6463
6564 # Chunksize here is for multiprocessing, not Zarr chunksize.
6665 pool_chunksize , extra = divmod (num_chunks , num_workers * 4 )
6766 pool_chunksize += 1 if extra else 0
6867
6968 tqdm_kw = {"unit" : "block" , "dynamic_ncols" : True }
7069
70+ # For Unix async writes with s3fs/fsspec & multiprocessing, use 'spawn' instead of default
71+ # 'fork' to avoid deadlocks on cloud stores. Slower but necessary. Default on Windows
72+ context = mp .get_context ("spawn" )
73+
7174 # Launch multiprocessing pool
7275 with ProcessPoolExecutor (max_workers = num_workers , mp_context = context ) as executor :
7376 lazy_work = executor .map (
You can’t perform that action at this time.
0 commit comments