Custom Preprocessors

[lucianchauvin]

Hello. I have created a custom pre-processor, and I am curious how I should go about using it with an existing model. Currently, if I just pass it into the model with the pre_processor flag, it obviously fails as it overwrites the model's required preprocessor. I get that the way to use it is to just compose the transforms, but I can't get this working. So, what is the intended way for an end user to use a custom preprocessor written by someone else with an existing model?

[samet-akcay]

@lucianchauvin, passing it to the model should ideally be the way. Can you provide minimal reproducible example?

[lucianchauvin]

Passing it to the model overwrites the model's preprocessor, as noted as one of the most common mistakes in the documentation. I provide an example below, but my preprocessors can be found here . You can see I'm trying to maintain the model's preprocessor by composing it with my transform. I am just curious how I should go about making a preprocessor that can be passed into any model (obviously, it must keep the dimensions the same).

Or maybe I misunderstand what a preprocessor is? Thanks for the help!

[alexriedel1]

Can you show the error log?
The issue might not be related to anomalib however.

[lucianchauvin]

Here is the bare minimum, where I simply take the model's preprocessor transform and apply it in my preprocessor instead. I also include the error I am getting.

from anomalib.pre_processing import PreProcessor
from anomalib.pre_processing.utils.transform import get_exportable_transform
from torchvision.transforms.v2 import Transform, Compose
import torch

class Default_Transform(Transform):
    def __init__(self, model_transform):
        super().__init__()
        print(model_transform)
        self.pre_transforms = Compose([
            model_transform
        ])

    def _transform(self, inpt: torch.tensor, params=None):
        original_device = inpt.device
        original_dtype = inpt.dtype

        processed_inputs = [self.pre_transforms(img) for img in inpt]
        processed_inputs = torch.stack(processed_inputs)
        
        return processed_inputs.to(original_device).to(original_dtype)

class Default(PreProcessor):
    def __init__(self, model_transform):
        super().__init__()
        self.transform = Default_Transform(model_transform)

    def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_val_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_test_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_predict_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

from anomalib.engine import Engine
from anomalib.models import Padim
from anomalib.data import MVTecAD

datamodule = MVTecAD(
    category="bottle",  # MVTec category to use
    train_batch_size=32,  # Number of images per training batch
    eval_batch_size=32,  # Number of images per validation/test batch
    num_workers=8,  # Number of parallel processes for data loading
)
datamodule.prepare_data()
datamodule.setup()

i, train_data = next(enumerate(datamodule.train_dataloader()))
print("Batch Image Shape", train_data.image.shape)
model = Padim(pre_processor=Default(model_transform = Padim.configure_pre_processor().transform))

engine = Engine()
engine.fit(model=model, datamodule=datamodule)
test_results = engine.test(
    model=model,
    datamodule=datamodule,
    ckpt_path=engine.trainer.checkpoint_callback.best_model_path,
)

### gives the following error
Traceback (most recent call last): 
  File "/home/lucian/src/SARIAD/demo/train.py", line 43, in <module>
    engine.fit(model=model, datamodule=datamodule)
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/anomalib/engine/engine.py", line 416, in fit
    self.trainer.fit(model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 539, in fit
    call._call_and_handle_interrupt(
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 47, in _call_and_handle_interrupt
    return trainer_fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 575, in _fit_impl
    self._run(model, ckpt_path=ckpt_path)
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 982, in _run
    results = self._run_stage()
              ^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 1026, in _run_stage
    self.fit_loop.run()
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/loops/fit_loop.py", line 216, in run
    self.advance()
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/loops/fit_loop.py", line 455, in advance
    self.epoch_loop.run(self._data_fetcher)
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 151, in run
    self.on_advance_end(data_fetcher)
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 370, in on_advance_end
    self.val_loop.run()
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/loops/utilities.py", line 179, in _decorator
    return loop_run(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 144, in run
    self._evaluation_step(batch, batch_idx, dataloader_idx, dataloader_iter)
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 422, in _evaluation_step
    call._call_callback_hooks(trainer, hook_name, *hook_kwargs.values())
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 222, in _call_callback_hooks
    fn(trainer, trainer.lightning_module, *args, **kwargs)
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/anomalib/pre_processing/pre_processor.py", line 103, in on_validation_batch_start
    batch.image, batch.gt_mask = self.transform(batch.image, batch.gt_mask)
                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/_transform.py", line 50, in forward
    flat_outputs = [
                   ^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/_transform.py", line 51, in <listcomp>
    self._transform(inpt, params) if needs_transform else inpt
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/SARIAD/pre_processing/Default/Default.py", line 19, in _transform
    processed_inputs = [self.pre_transforms(img) for img in inpt]
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/SARIAD/pre_processing/Default/Default.py", line 19, in <listcomp>
    processed_inputs = [self.pre_transforms(img) for img in inpt]
                        ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/_container.py", line 51, in forward
    outputs = transform(*inputs)
              ^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/_container.py", line 51, in forward
    outputs = transform(*inputs)
              ^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/_transform.py", line 50, in forward
    flat_outputs = [
                   ^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/_transform.py", line 51, in <listcomp>
    self._transform(inpt, params) if needs_transform else inpt
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/_geometry.py", line 160, in _transform
    return self._call_kernel(
           ^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/_transform.py", line 35, in _call_kernel
    return kernel(inpt, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/lucian/src/SARIAD/venv/lib/python3.11/site-packages/torchvision/transforms/v2/functional/_geometry.py", line 226, in resize_image
    num_channels, old_height, old_width = shape[-3:]
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ValueError: not enough values to unpack (expected 3, got 2)

Thanks for the help!

[lucianchauvin]

@samet-akcay and ideas on this? I would appreciate any help!

[alfieroddan]

Your Default_Transform class does not return the right values.

Here are the Docs in implementing your own v2 transforms.

https://docs.pytorch.org/vision/main/auto_examples/transforms/plot_custom_transforms.html#sphx-glr-auto-examples-transforms-plot-custom-transforms-py

Below I've re-implemented your code in a different way.

[alfieroddan]

Hello,

This works for me just fine, maybe it'll help you:

from anomalib.engine import Engine
from anomalib.models import Padim
from anomalib.data import MVTecAD
from anomalib.pre_processing import PreProcessor
from torchvision.transforms.v2 import RandomVerticalFlip, Compose


class DefaultProcessor(PreProcessor):
    def __init__(self, transform):
        super().__init__(transform=transform)

    def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_val_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_test_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_predict_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)


datamodule = MVTecAD(
    category="bottle",  # MVTec category to use
    train_batch_size=32,  # Number of images per training batch
    eval_batch_size=32,  # Number of images per validation/test batch
    num_workers=8,  # Number of parallel processes for data loading
)
datamodule.prepare_data()
datamodule.setup()

i, train_data = next(enumerate(datamodule.train_dataloader()))
print("Batch Image Shape", train_data.image.shape)

new_transform = RandomVerticalFlip(p=1.0)
old_transform = Padim.configure_pre_processor().transform
combined_transform = Compose(
    [new_transform, *old_transform.transforms],
)
default_preprocess = DefaultProcessor(transform=combined_transform)
model = Padim(pre_processor=default_preprocess)

engine = Engine()
engine.fit(model=model, datamodule=datamodule)
test_results = engine.test(
    model=model,
    datamodule=datamodule,
    ckpt_path=engine.trainer.checkpoint_callback.best_model_path,
)

You can see I'm trying to maintain the model's preprocessor by composing it with my transform

You are also overwriting the model's pre-processor but just keeping the same transforms.

You could in theory in-place change the `PreProcessor``s transforms, see here. But I recommend pre-pending or appending. Like above.

[lucianchauvin]

Thank you for your help! I guess what I am really stuck on is being able to do all of this

new_transform = RandomVerticalFlip(p=1.0)
old_transform = Padim.configure_pre_processor().transform
combined_transform = Compose(
    [new_transform, *old_transform.transforms],
)
default_preprocess = DefaultProcessor(transform=combined_transform)

within my preprocessor. I am trying to make a custom preprocessor that uses my own transform whilst preserving the models' required transforms. I want something like this but cant even get just applying the default transforms to work,

from anomalib.pre_processing import PreProcessor
from anomalib.pre_processing.utils.transform import get_exportable_transform
from torchvision.transforms.v2 import Transform, Compose
import torch

class Default_Transform(Transform):
    def __init__(self, model):
        super().__init__()
        self.pre_transform = Compose([
            *model.configure_pre_processor().transform.transforms, 
        ])

    def transform(self, inpt: torch.tensor, params=None):
        input = self.pre_transform(inpt)
        # maybe so more stuff I want to do for a custom transform
        return input + 1

class Default(PreProcessor):
    def __init__(self, model_transform):
        super().__init__()
        self.transform = Default_Transform(model_transform)

    def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_validation_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_test_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_predict_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

from anomalib.engine import Engine
from anomalib.models import Padim
from anomalib.data import MVTecAD

datamodule = MVTecAD(
    category="bottle",  # MVTec category to use
    train_batch_size=32,  # Number of images per training batch
    eval_batch_size=32,  # Number of images per validation/test batch
    num_workers=8,  # Number of parallel processes for data loading
)
datamodule.prepare_data()
datamodule.setup()

i, train_data = next(enumerate(datamodule.train_dataloader()))
print("Batch Image Shape", train_data.image.shape)
model = Padim(pre_processor=Default(Padim))

engine = Engine()
engine.fit(model=model, datamodule=datamodule)
test_results = engine.test(
    model=model,
    datamodule=datamodule,
    ckpt_path=engine.trainer.checkpoint_callback.best_model_path,
)

[samet-akcay]

@lucianchauvin, which model do you use? Have you experienced this with all of the models, or some specific ones?

[lucianchauvin]

@lucianchauvin, which model do you use? Have you experienced this with all of the models, or some specific ones?

In all my examples I'm using PaDiM. But once again this is more of "how to go about implementing this within my own preprocessor" hence the example above. I don't think any of the issues have been model specific, but I can try a different one at some point.

[alfieroddan]

It's unorthodox but this works for me 😄:

from anomalib.pre_processing import PreProcessor
from anomalib.pre_processing.utils.transform import get_exportable_transform
from torchvision.transforms.v2 import Transform, Compose
import torch

class Default_Transform(Transform):
    def __init__(self, model_transform):
        super().__init__()
        self.pre_transform = model_transform

    def transform(self, inpt: torch.tensor, params=None):
        input = self.pre_transform(inpt)
        # maybe so more stuff I want to do for a custom transform
        return input + 1

class Default(PreProcessor):
    def __init__(self, model_transform):
        super().__init__()
        self.transform = Default_Transform(model_transform)

    def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_validation_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_test_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

    def on_predict_batch_start(self, trainer, pl_module, batch, batch_idx):
        batch.image = self.transform(batch.image)

from anomalib.engine import Engine
from anomalib.models import Padim
from anomalib.data import MVTecAD

datamodule = MVTecAD(
    category="bottle",  # MVTec category to use
    train_batch_size=32,  # Number of images per training batch
    eval_batch_size=32,  # Number of images per validation/test batch
    num_workers=8,  # Number of parallel processes for data loading
)
datamodule.prepare_data()
datamodule.setup()

i, train_data = next(enumerate(datamodule.train_dataloader()))
print("Batch Image Shape", train_data.image.shape)
model = Padim(pre_processor=Default(Padim.configure_pre_processor().transform))

engine = Engine()
engine.fit(model=model, datamodule=datamodule)
test_results = engine.test(
    model=model,
    datamodule=datamodule,
    ckpt_path=engine.trainer.checkpoint_callback.best_model_path,
)

Is this or the previous not what you're looking for?