AttributeError: Can't get attribute 'DataReader' on <module '__main__' (built-in)>

[talhaanwarch]

Here is DataReader

class DataReader(torch.utils.data.Dataset):
  def __init__(self, df):
        super(DataReader,self).__init__()
        self.df = df
  def __len__(self):
        'Denotes the total number of samples'
        return len(self.df)

  def __getitem__(self, index):
        'Generates one sample of data'
        # Select sample
        file = self.df.iloc[index,0]
        label=self.df.iloc[index,1]
        return data,label

when i do

x=DataLoader(DataReader(train), batch_size = 2,collate_fn=my_collate)
b=next(iter(x))

it worked. but when I call it inside LightningModule, it throws an error

class OurModel(LightningModule):
  def __init__(self):
    super(OurModel,self).__init__()
    self.model =cnnmodel()
  def forward(self,x):
    x= self.model(x)
    return x

  def configure_optimizers(self):
    return torch.optim.AdamW(params=self.parameters(),lr=self.lr )

  def train_dataloader(self):
    return DataLoader(DataReader(train))

  def training_step(self,batch,batch_idx):
    return loss

  def val_dataloader(self):
    ds= DataLoader(DataReader(val))
    print('ds',len(ds))
    return
    
  def validation_step(self,batch,batch_idx):
    print('val step')
    return 

I am unable to figure out what is the error and how to resolve it.I tried to debug it, val_dataloader function is working, it print ds 7 but validation_step is not working, it should print val step, but its not printing it

EDIT
This issue is because of numworker, setting numworker to 0, resolve the issue, but I am getting this warning
Consider increasing the value of thenum_workers argument (try 12 which is the number of cpus on this machine)`

Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Anaconda3\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\Anaconda3\lib\multiprocessing\spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
AttributeError: Can't get attribute 'DataReader' on <module '__main__' (built-in)>
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Anaconda3\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\Anaconda3\lib\multiprocessing\spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
AttributeError: Can't get attribute 'DataReader' on <module '__main__' (built-in)>
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Anaconda3\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\Anaconda3\lib\multiprocessing\spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
AttributeError: Can't get attribute 'DataReader' on <module '__main__' (built-in)>
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Anaconda3\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\Anaconda3\lib\multiprocessing\spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
AttributeError: Can't get attribute 'DataReader' on <module '__main__' (built-in)>
Traceback (most recent call last):

  File "C:\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 872, in _try_get_data
    data = self._data_queue.get(timeout=timeout)

  File "C:\Anaconda3\lib\multiprocessing\queues.py", line 108, in get
    raise Empty

Empty


The above exception was the direct cause of the following exception:

Traceback (most recent call last):

  File "<ipython-input-9-5ebd04eb49d1>", line 101, in <module>
    trainer.fit(model)

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 458, in fit
    self._run(model)

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 756, in _run
    self.dispatch()

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 797, in dispatch
    self.accelerator.start_training(self)

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\accelerators\accelerator.py", line 96, in start_training
    self.training_type_plugin.start_training(trainer)

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\plugins\training_type\training_type_plugin.py", line 144, in start_training
    self._results = trainer.run_stage()

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 807, in run_stage
    return self.run_train()

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 842, in run_train
    self.run_sanity_check(self.lightning_module)

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 1107, in run_sanity_check
    self.run_evaluation()

  File "C:\Anaconda3\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 949, in run_evaluation
    for batch_idx, batch in enumerate(dataloader):

  File "C:\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 435, in __next__
    data = self._next_data()

  File "C:\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 1068, in _next_data
    idx, data = self._get_data()

  File "C:\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 1034, in _get_data
    success, data = self._try_get_data()

  File "C:\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 885, in _try_get_data
    raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) from e

RuntimeError: DataLoader worker (pid(s) 4984, 20904) exited unexpectedly

[carmocca]

You can disable that warning with the following

import warnings
warnings.filterwarnings("ignore", message=".*does not have many workers which may be a bottleneck.*")

[talhaanwarch]

but cant we use more num_workers if the system has? i am using windows, I thinks its because of this

[carmocca]

What environment are you on? Is it a notebook?

How are you creating your trainer?

[tchaton]

Did you try to add

if name == "main"

to your script ?

[talhaanwarch]

do i need to write whole code under `if __name__ == "__main__"` or just `trainer.fit(model)`

…

On Mon, Aug 16, 2021 at 2:00 PM thomas chaton ***@***.***> wrote: Did you try to add if *name* == "*main*" to your script ? — You are receiving this because you authored the thread. Reply to this email directly, view it on GitHub <#8898 (reply in thread)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AI5FYO3X4RDTD7LMWYTWCBTT5DHTXANCNFSM5CEXIGTA> .

[talhaanwarch]

I figured it out. Instead of spyder use CMD
and move code under main like this

if __name__ == '__main__':
    es=EarlyStopping(monitor="val_loss",patience=20, verbose=True,mode='min')
    lr_monitor = LearningRateMonitor(logging_interval='epoch')
    checkpoint_callback = ModelCheckpoint(monitor='val_acc',filename='chkpnt')
    model = OurModel()
    trainer = Trainer(max_epochs=500, auto_lr_find=False, auto_scale_batch_size=False,
                      gpus=-1,
                      progress_bar_refresh_rate=0, callbacks=[es,lr_monitor,checkpoint_callback]
                      )        
    trainer.fit(model)
    trainer.test(model)