core dumped Slurm pytorch

[foguy]

Hi,
I am new in pytorch lighting, I want to run my code in slurm cluster, but i get this error

Training: 0it [00:00, ?it/s]
Training:   0% 0/16670 [00:00<?, ?it/s]
Epoch 0:   0% 0/16670 [00:00<?, ?it/s] /var/spool/slurmd/job60005806/slurm_script: line 36: 20744 Aborted                 (core dumped) 

This is part of my code :

def main(path=None, config=None, save_top_k=-1,resume_from_checkpoint=None, gpus=1, seed=22, result_file=None, fast_dev_run= False,  seq_length=10, epochs=10,  num_layers = 2,  lr = 0.1, weight_decay = 0.005, val_rate=0.2, feature=None,   model_name='debug',  cell=None):
    seed_everything(seed)
    
    hp = OmegaConf.load(config)
    network =  ConvLSTMNetworkSE(img_size, hp.hpara.input_channel, hidden_size, kernel_size, stride, padding, num_layers,  nb_classes=36,
                             Used_SE_block= hp.hpara.SE_block, bidirectional = True)
    #network = ConvGRUNetworkSE(img_size, input_channel, hidden_size, kernel_size, stride, padding, 
    #                       num_layers, nb_classes=36,  Used_SE_block=None, bidirectional = True)

            
    model =  ConvTempAttent(network, hp)
    print(model)
      
    for i in range(len(list(model.parameters()))):
        print(list(model.parameters())[i].size())

    save_path = os.path.join(hp.arclulc.chkpt_dir, model_name)
    os.makedirs(save_path, exist_ok=True)
    

    # ------------
    # Instantiate the LightningDataModule
    # ------------
    data_module = LULC_Classif_Data(path, hp, feature, seq_length, val_rate, result_file)
    
    # Load the DataLoaders for both the train and validation datasets
    train_loader = data_module.train_dataloader()
    val_loader = data_module.val_dataloader()
        
    # ------------
    # logging wandb
    # ------------
    # This logger is used when we call self.log inside the LightningModule
        # define logger
    #https://pytorch-lightning.readthedocs.io/en/stable/extensions/generated/pytorch_lightning.loggers.WandbLogger.html
    wandb_logger = WandbLogger(project=hp.wandb.project, entity=hp.wandb.entity, log_model="True")

    wandb_logger.watch(model)

    early_stop_callback = EarlyStopping(
       monitor='val/loss',
       min_delta=1e-4,
       patience= hp.hpara.patience,
       verbose=False,
       mode='min'
        )


    checkpoint_callback = ModelCheckpoint(
        dirpath=os.path.join(model_name),
        monitor='val/loss`',
        verbose=True,
        save_top_k=save_top_k,  # save all
         )


    # Use as many GPUs as are available
    #gpus = -1 if gpus is None else gpus,

    # ------------
    # training
    # ------------

    trainer = pl.Trainer(
    benchmark=hp.hpara.benchmark,  # cudnn.benchmark
    callbacks= [early_stop_callback],
    checkpoint_callback=checkpoint_callback,
    weights_save_path=save_path,
    check_val_every_n_epoch=hp.hyper.val_interval,
    deterministic=hp.hpara.deterministic,  # cudnn.deterministic
    gpus = 1 if torch.cuda.is_available() else None,
    resume_from_checkpoint=None,
    gradient_clip_val=hp.hyper.grad_clip,#0.1
    progress_bar_refresh_rate=1,
    max_epochs=epochs,
    num_sanity_val_steps=0,
    accelerator='dp',
    logger=wandb_logger)  

    trainer.fit(model, train_loader, val_loader)