error caused by T.ToDeviced(keys=["img", "seg"],device=cfg.device)

[lg2578]

when I tried to load cachedataset to GPU,there is an error.
error caused by T.ToDeviced(keys=["img", "seg"],device=cfg.device)
cfg.device is cuda:0
my code:

import monai.transforms as T
train_transforms = T.Compose(
    [
        T.LoadImaged(keys=["img", "seg"]),
        T.AddChanneld(keys=["img", "seg"]),
        T.ScaleIntensityd(keys=["img", "seg"]),
        T.RandSpatialCropd(keys=["img", "seg"],roi_size=cfg.roi_size,random_size=False),
        T.EnsureTyped(keys=["img", "seg"]),
        T.ToDeviced(keys=["img", "seg"],device=cfg.device)
    ]
)
# define dataset, data loader
from monai.data import CacheDataset, DataLoader, Dataset, decollate_batch
from monai.data import list_data_collate,  decollate_batch,pad_list_data_collate
check_ds = monai.data.Dataset(data=train_files, transform=train_transforms)
# use batch_size=2 to load images and use RandCropByPosNegLabeld to generate 2 x 4 images for network training
check_loader = monai.data.DataLoader(check_ds, batch_size=2, num_workers=2, collate_fn=pad_list_data_collate)
check_data = monai.utils.misc.first(check_loader)
print(check_data["img"].shape, check_data["seg"].shape)

errors:

RuntimeError Traceback (most recent call last)
/tmp/ipykernel_33/3130994972.py in
5 # use batch_size=2 to load images and use RandCropByPosNegLabeld to generate 2 x 4 images for network training
6 check_loader = monai.data.DataLoader(check_ds, batch_size=2, num_workers=2, collate_fn=pad_list_data_collate)
----> 7 check_data = monai.utils.misc.first(check_loader)
8 print(check_data["img"].shape, check_data["seg"].shape)
9

/kaggle/input/monai-v081/monai/utils/misc.py in first(iterable, default)
80 Returns the first item in the given iterable or default if empty, meaningful mostly with 'for' expressions.
81 """
---> 82 for i in iterable:
83 return i
84 return default

/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py in next(self)
528 if self._sampler_iter is None:
529 self._reset()
--> 530 data = self._next_data()
531 self._num_yielded += 1
532 if self._dataset_kind == _DatasetKind.Iterable and \

/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py in _next_data(self)
1222 else:
1223 del self._task_info[idx]
-> 1224 return self._process_data(data)
1225
1226 def _try_put_index(self):

/opt/conda/lib/python3.7/site-packages/torch/utils/data/dataloader.py in _process_data(self, data)
1248 self._try_put_index()
1249 if isinstance(data, ExceptionWrapper):
-> 1250 data.reraise()
1251 return data
1252

/opt/conda/lib/python3.7/site-packages/torch/_utils.py in reraise(self)
455 # instantiate since we don't know how to
456 raise RuntimeError(msg) from None
--> 457 raise exception
458
459

RuntimeError: Caught RuntimeError in DataLoader worker process 0.
Original Traceback (most recent call last):
File "../input/monai-v081/monai/transforms/transform.py", line 89, in apply_transform
return _apply_transform(transform, data, unpack_items)
File "../input/monai-v081/monai/transforms/transform.py", line 53, in _apply_transform
return transform(parameters)
File "../input/monai-v081/monai/transforms/utility/dictionary.py", line 1550, in call
d[key] = self.converter(d[key])
File "../input/monai-v081/monai/transforms/utility/array.py", line 1213, in call
return img.to(self.device, **self.kwargs)
File "/opt/conda/lib/python3.7/site-packages/torch/cuda/init.py", line 207, in _lazy_init
"Cannot re-initialize CUDA in forked subprocess. To use CUDA with "
RuntimeError: Cannot re-initialize CUDA in forked subprocess. To use CUDA with multiprocessing, you must use the 'spawn' start method

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "../input/monai-v081/monai/transforms/transform.py", line 89, in apply_transform
return _apply_transform(transform, data, unpack_items)
File "../input/monai-v081/monai/transforms/transform.py", line 53, in apply_transform
return transform(parameters)
File "../input/monai-v081/monai/transforms/compose.py", line 173, in call
input = apply_transform(transform, input, self.map_items, self.unpack_items, self.log_stats)
File "../input/monai-v081/monai/transforms/transform.py", line 113, in apply_transform
raise RuntimeError(f"applying transform {transform}") from e
RuntimeError: applying transform <monai.transforms.utility.dictionary.ToDeviced object at 0x7f9224f6e190>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
data = fetcher.fetch(index)
File "/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 49, in
data = [self.dataset[idx] for idx in possibly_batched_index]
File "../input/monai-v081/monai/data/dataset.py", line 97, in getitem
return self._transform(index)
File "../input/monai-v081/monai/data/dataset.py", line 83, in _transform
return apply_transform(self.transform, data_i) if self.transform is not None else data_i
File "../input/monai-v081/monai/transforms/transform.py", line 113, in apply_transform
raise RuntimeError(f"applying transform {transform}") from e
RuntimeError: applying transform <monai.transforms.compose.Compose object at 0x7f9224f6e910>

[Nic-Ma]

Hi @lg2578 ,

As your error log shows: RuntimeError: Cannot re-initialize CUDA in forked subprocess. To use CUDA with multiprocessing, you must use the 'spawn' start method
CUDA can't work with forked multi-processing, please set the num_workers of DataLoader to 0.
Suggest to use ThreadDataLoader for this case, refer to:
https://github.com/Project-MONAI/tutorials/blob/main/acceleration/distributed_training/brats_training_ddp.py#L220

Thanks.

[lg2578]

Hi @Nic-Ma , Thank you very much.It works