Creating Datamodule/ Training with custom data loaders for k-Cross Validation

[simon-ami]

Describe the bug

Hi,

while trying to find a way to implement kfold cross validation i'm facing the issue that i can't get checkpointing to work with custom dataloaders. The same config without the custom loaders additionally added works fine, with checkpoints.

I have tried directly passing the loaders to the fit method and also tried appending the dataloaders to the datamodule and passing that to the fit() method too, but it won't work.

Below is my function used:

def k_fold_cross_validation(config, k=5):
    """
    Perform k-fold cross-validation on the given data module and model.

    Args:
        config (OmegaConf): The configuration object.
        model (torch.nn.Module): The PyTorch model to be trained and evaluated.
        k (int, optional): The number of folds for cross-validation. Defaults to 5.

    Returns:
        mean_f1_score, mean_auroc_score, std_f1_score, std_auroc_score, avg_time_per_image
    """

    curr_dir = os.path.dirname(os.path.abspath(__file__))
    print(f"Current directory: {curr_dir}")
    model_name = config.model.name
    time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
    checkpoint_dir = curr_dir + f"./../../data/results/{model_name}/vespa/run/weights/"
    checkpoint_dir = os.path.abspath(checkpoint_dir + time)
    print(f"Checkpoint directory: {checkpoint_dir}")

    f1_scores = []
    auroc_scores = []
    best_model_paths = []
    time_per_fold = []   # time for prediction per image

    data_module = get_datamodule(config)
    data_module.setup()
    print(f"Length of train data: {len(data_module.train_data)},\n Length of test data: {len(data_module.test_data)}")
    print(f"Train has normal: {data_module.train_data.has_normal}")
    print(f"Train has anomalous: {data_module.train_data.has_anomalous}")
    print(f"Test has normal: {data_module.test_data.has_normal}")
    print(f"Test has anomalous: {data_module.test_data.has_anomalous}")

    # Combine train and test data
    normal_data = data_module.train_data
    anomalous_data = data_module.test_data

    # Create k-fold cross-validator
    kfold = KFold(n_splits=k, shuffle=True, random_state=data_module.seed)

    # Perform k-fold cross-validation
    for fold, (train_indices, test_indices) in enumerate(kfold.split(normal_data)):
        print(f"Fold {fold + 1} of {k}")

        # Create train and test data loaders
        train_set = torch.utils.data.Subset(normal_data, train_indices)
        train_loader = DataLoader(train_set,
                                  batch_size=data_module.train_batch_size,
                                  num_workers=data_module.num_workers)
        test_normal_set = torch.utils.data.Subset(normal_data, test_indices)
        test_anomalous_set = torch.utils.data.Subset(anomalous_data, test_indices)
        combined_test_set = ConcatDataset([test_normal_set, test_anomalous_set])
                                        
        test_loader = DataLoader(combined_test_set,
                                 batch_size=data_module.eval_batch_size,
                                 num_workers=data_module.num_workers)
                                 
        data_module.train_dataloader = train_loader
        data_module.val_dataloader = test_loader
        
        # check number of anomalous and normal images in each set
        print(f"Train has n normal images: {sum([1 for i in train_set if i['label'] == 0])}")
        print(f"Train has n anomalous images: {sum([1 for i in train_set if i['label'] == 1])}")
        print(f"Test has n normal images: {sum([1 for i in combined_test_set if i['label'] == 0])}")
        print(f"Test has n anomalous images: {sum([1 for i in combined_test_set if i['label'] == 1])}")

        # Train the model
        model = m.get_model(config)

        # Create a logger and callbacks
        experiment_logger = get_experiment_logger(config)
        callbacks = get_callbacks(config)
        print(f"DEBUG BEFORE CheckpointCallback: {callbacks[0]}")
        # remove orig modelcheckpoint callback and add custom one
        # callbacks = callbacks[1:]
        # checkpoint_callback = ModelCheckpoint(
        #     dirpath=checkpoint_dir,
        #     filename="model",
        #     monitor="image_AUROC",
        #     mode="max",
        #     verbose=True,
        #     every_n_epochs=1,
        # )
        # callbacks.insert(0, checkpoint_callback)

        trainer = m.get_trainer(config, experiment_logger, callbacks)
        print(f"DEBUG NEW CheckpointCallback: {trainer.checkpoint_callback}")
        trainer.fit(model, datamodule=data_module)

        # get best model path
        best_model_path=trainer.checkpoint_callback.best_model_path
        load_model_callback = LoadModelCallback(
            weights_path=best_model_path,
        )
        trainer.callbacks.insert(0, load_model_callback)
        print(f"Best model path: {best_model_path}")

        # load best model
        model = m.get_model(config)
        model.load_state_dict(torch.load(best_model_path)["state_dict"])

        # evaluate the model on the test set
        start_time = time.time()
        trainer.test(model, test_loader)
        end_time = time.time()

        f1_score = trainer.logged_metrics["image_F1Score"]
        auroc_score = trainer.logged_metrics["image_AUROC"]
        f1_scores.append(f1_score.item())
        auroc_scores.append(auroc_score.item())
        best_model_paths.append(best_model_path)
        time_per_fold.append(end_time - start_time)

    # Calculate the average performance metrics and standard deviation
    mean_f1_score = statistics.mean(f1_scores)
    mean_auroc_score = statistics.mean(auroc_scores)
    std_f1_score = statistics.stdev(f1_scores)
    std_auroc_score = statistics.stdev(auroc_scores)
    # Calculate the average time per image
    avg_time_per_image = sum(time_per_fold) / (k * len(data_module.test_data))    

    return mean_f1_score, mean_auroc_score, std_f1_score, std_auroc_score, best_model_paths, avg_time_per_image

Dataset

Folder

Model

Reverse Distillation

Steps to reproduce the behavior

1. install anomalib 040
2. copy config
3. use own custom dataset or some other
4. run my kfold cv function with the loaded config
5. see error

OS information

OS information:

• OS: Win 10
• Python version: 3.8.16
• Anomalib version: 0.4.0
• PyTorch version: 1.12.1+cu116
• CUDA/cuDNN version: 11.6/ 8302
• GPU models and configuration: GeForce RTX 3050
• Any other relevant information: I'm using a custom dataset

Expected behavior

the model callback should create checkpoints every epoch

Screenshots

No response

Pip/GitHub

pip

What version/branch did you use?

0.4.0

Configuration YAML

dataset:
  abnormal_dir: bad
  center_crop: null
  extensions: null
  format: folder
  image_size: 224
  mask: null
  name: vespa
  normal_dir: good
  normal_test_dir: null
  normalization: imagenet
  num_workers: 8
  path: ./../data/images/data_structure_anomalib
  split_ratio: 0.2
  task: classification
  test_batch_size: 8
  test_split_mode: from_dir
  test_split_ratio: 0.2
  tiling:
    apply: false
    random_tile_count: 16
    remove_border_count: 0
    stride: null
    tile_size: null
    use_random_tiling: false
  train_batch_size: 8
  transform_config:
    eval: C:\Users\benedict\dev\VespaUseCase\machine_learning\src\d01_config\transform_config_1.yaml
    train: C:\Users\benedict\dev\VespaUseCase\machine_learning\src\d01_config\transform_config_1.yaml
  val_split_mode: same_as_test
  val_split_ratio: 0.3
logging:
  log_graph: false
  logger:
  - tensorboard
metrics:
  image:
  - F1Score
  - AUROC
  pixel:
  - F1Score
  - AUROC
  threshold:
    manual_image: null
    manual_pixel: null
    method: adaptive
model:
  anomaly_map_mode: multiply
  backbone: wide_resnet50_2
  beta1: 0.5
  beta2: 0.99
  early_stopping:
    metric: image_AUROC
    mode: max
    patience: 20
  layers:
  - layer1
  - layer2
  - layer3
  lr: 0.0002
  name: reverse_distillation
  normalization_method: min_max
  pre_trained: true
optimization:
  export_mode: null
project:
  path: ./../data/results
  seed: 42
trainer:
  accelerator: auto
  accumulate_grad_batches: 1
  auto_lr_find: true
  auto_scale_batch_size: false
  benchmark: false
  check_val_every_n_epoch: 2
  default_root_dir: null
  detect_anomaly: false
  deterministic: false
  devices: 1
  enable_checkpointing: true
  enable_model_summary: true
  enable_progress_bar: true
  fast_dev_run: false
  gradient_clip_algorithm: norm
  gradient_clip_val: 0
  limit_predict_batches: 1.0
  limit_test_batches: 1.0
  limit_train_batches: 1.0
  limit_val_batches: 1.0
  log_every_n_steps: 8
  max_epochs: 200
  max_steps: -1
  min_epochs: null
  min_steps: null
  move_metrics_to_cpu: false
  multiple_trainloader_mode: max_size_cycle
  num_nodes: 1
  num_sanity_val_steps: 0
  overfit_batches: 0.0
  plugins: null
  precision: 32
  profiler: null
  reload_dataloaders_every_n_epochs: 0
  replace_sampler_ddp: true
  strategy: null
  sync_batchnorm: false
  tpu_cores: null
  track_grad_norm: -1
  val_check_interval: 1.0
visualization:
  image_save_path: ./../data/results/images
  log_images: true
  mode: full
  save_images: true
  show_images: false
  task: classification

Logs

AttributeError                            Traceback (most recent call last)
Cell In[6], line 2
      1 # cross validate the model
----> 2 mean_f1_score, mean_auroc_score, std_f1_score, std_auroc_score, best_model_paths, avg_time_per_image = e.k_fold_cross_validation(config, k=2)

File [c:\Users\benedict\dev\VespaUseCase\machine_learning\src/../../../VespaUseCase\machine_learning\src\d06_evaluation\evaluation.py:278](file:///C:/Users/benedict/dev/VespaUseCase/machine_learning/src/d06_evaluation/evaluation.py:278), in k_fold_cross_validation(config, k)
    276 trainer = m.get_trainer(config, experiment_logger, callbacks)
    277 print(f"DEBUG NEW CheckpointCallback: {trainer.checkpoint_callback}")
--> 278 trainer.fit(model, datamodule=data_module)
    280 # get best model path
    281 best_model_path=trainer.checkpoint_callback.best_model_path

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\pytorch_lightning\trainer\trainer.py:608](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/pytorch_lightning/trainer/trainer.py:608), in Trainer.fit(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)
    606 model = self._maybe_unwrap_optimized(model)
    607 self.strategy._lightning_module = model
--> 608 call._call_and_handle_interrupt(
    609     self, self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path
    610 )

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\pytorch_lightning\trainer\call.py:38](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/pytorch_lightning/trainer/call.py:38), in _call_and_handle_interrupt(trainer, trainer_fn, *args, **kwargs)
     36         return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
     37     else:
---> 38         return trainer_fn(*args, **kwargs)
     40 except _TunerExitException:
     41     trainer._call_teardown_hook()

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\pytorch_lightning\trainer\trainer.py:638](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/pytorch_lightning/trainer/trainer.py:638), in Trainer._fit_impl(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)
    633     raise MisconfigurationException(
    634         "You cannot pass `train_dataloader` or `val_dataloaders` to `trainer.fit(datamodule=...)`"
    635     )
    637 # links data to the trainer
--> 638 self._data_connector.attach_data(
    639     model, train_dataloaders=train_dataloaders, val_dataloaders=val_dataloaders, datamodule=datamodule
    640 )
    642 # TODO: ckpt_path only in v2.0
    643 ckpt_path = ckpt_path or self.resume_from_checkpoint

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\pytorch_lightning\trainer\connectors\data_connector.py:139](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:139), in DataConnector.attach_data(self, model, train_dataloaders, val_dataloaders, test_dataloaders, predict_dataloaders, datamodule)
    137 # Validate that the required data sources are available
    138 if self.trainer.state.fn == TrainerFn.FITTING:
--> 139     _check_dataloader_none(train_dataloaders, self._train_dataloader_source, self.trainer.state.fn)
    140 elif self.trainer.state.fn == TrainerFn.VALIDATING:
    141     _check_dataloader_none(val_dataloaders, self._val_dataloader_source, self.trainer.state.fn)

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\pytorch_lightning\trainer\connectors\data_connector.py:597](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:597), in _check_dataloader_none(dataloader, dataloader_source, trainer_fn)
    590 def _check_dataloader_none(
    591     dataloader: Optional[Union[TRAIN_DATALOADERS, EVAL_DATALOADERS]],
    592     dataloader_source: _DataLoaderSource,
    593     trainer_fn: TrainerFn,
    594 ) -> None:
    595     # A prefix in the message to disambiguate between the train- and (optional) val dataloader that .fit() accepts
    596     prefix = "train_" if trainer_fn == TrainerFn.FITTING else ""
--> 597     if dataloader is None and not dataloader_source.is_defined():
    598         raise ValueError(
    599             f"An invalid dataloader was passed to `Trainer.{trainer_fn}({prefix}dataloaders=...)`."
    600             f" Either pass the dataloader to the `.{trainer_fn}()` method OR implement"
    601             f" `def {dataloader_source.name}(self):` in your LightningModule/LightningDataModule."
    602         )

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\pytorch_lightning\trainer\connectors\data_connector.py:534](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:534), in _DataLoaderSource.is_defined(self)
    529 def is_defined(self) -> bool:
    530     """Returns whether the source dataloader can be retrieved or not.
    531 
    532     If the source is a module it checks that the method with given :attr:`name` is overridden.
    533     """
--> 534     return not self.is_module() or is_overridden(self.name, self.instance)

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\pytorch_lightning\utilities\model_helpers.py:37](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/pytorch_lightning/utilities/model_helpers.py:37), in is_overridden(method_name, instance, parent)
     34         raise ValueError("Expected a parent")
     35 from lightning_utilities.core.overrides import is_overridden
---> 37 return is_overridden(method_name, instance, parent)

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\lightning_utilities\core\overrides.py:32](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/lightning_utilities/core/overrides.py:32), in is_overridden(method_name, instance, parent)
     29 if parent_attr is None:
     30     raise ValueError("The parent should define the method")
---> 32 return instance_attr.__code__ != parent_attr.__code__

AttributeError: 'DataLoader' object has no attribute '__code__'

Code of Conduct

• I agree to follow this project's Code of Conduct

[samet-akcay]

You seem to pass a test dataloader to test the model here:

trainer.test(model, test_loader)

Based on the error message, PyTorch Lightning does not seem to like this.

File [c:\Users\benedict\anaconda3\envs\anomalib_env\lib\site-packages\pytorch_lightning\trainer\trainer.py:638](file:///C:/Users/benedict/anaconda3/envs/anomalib_env/lib/site-packages/pytorch_lightning/trainer/trainer.py:638), in Trainer._fit_impl(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)
    633     raise MisconfigurationException(
    634         "You cannot pass `train_dataloader` or `val_dataloaders` to `trainer.fit(datamodule=...)`"
    635     )

This is an interesting application; however the error stack you are getting is not an issue in anomalib. I will therefore move this to a Q&A in Discussions. Feel free to continue the discussion there.