How to use matplotlib for image drawing in MapTransform?

[Minxiangliu]

I love using MONAI's great modules, but have problems writing my own code. I want to convert the data into an image in Transform. I refer to some monai practices, but I encounter difficulties in matplotlib. If AxesSubplot.clear() is not executed, the data will be repeated on the figure. If executed, it will be something went wrong, how can I improve my code?
Thanks in advance!

Code:

import torch
import numpy as np
from monai.transforms.transform import MapTransform
from monai.data import CacheDataset, ThreadDataLoader
from monai.transforms import Compose
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas

class convertImage(MapTransform):
    def __init__(self,keys):
        super().__init__(keys)
        self.fig = plt.figure(figsize=(12,7), tight_layout=True, dpi=20)
        self.FigLits = list()
        for i in range(5,-1,-1):
            self.FigLits.append(self.fig.add_subplot(6,1,i+1))
        self.canvas = FigureCanvas(self.fig)
        
    def convertIMG(self, data):
        for i,graph in enumerate(self.FigLits):
            graph.clear()   #There is a problem here...
            graph.plot(np.arange(len(data)), data[:,i], linewidth=3)
            graph.axis('off')

        self.canvas.draw()
        width, height = self.fig.get_size_inches()*self.fig.get_dpi()
        image = np.frombuffer(self.canvas.tostring_rgb(), dtype='uint8').reshape(int(height), int(width), 3)
        R, G, B = image[:,:,0], image[:,:,1], image[:,:,2]
        imgGray = 0.2989 * R + 0.5870 * G + 0.1140 * B
        imgGray = imgGray.max() - imgGray
        return torch.tensor(imgGray).to(torch.int32)

    def __call__(self, data):
        d = dict(data)
        for key in self.key_iterator(d):
            d[key] = self.convertIMG(d[key].squeeze(0))
        return d

train_loader = ThreadDataLoader(CacheDataset(data=[{'img':torch.rand(1,200,6)}, 
                                                   {'img':torch.rand(1,200,6)},
                                                   {'img':torch.rand(1,200,6)},
                                                   {'img':torch.rand(1,200,6)}],
                                             transform=Compose(convertImage(keys=['img']))),                                
                                num_workers=0, 
                                batch_size=2, 
                                shuffle=True,
                                drop_last=True)
for d in train_loader:
    d

If AxesSubplot.clear() is not executed:

Otherwise

Traceback (most recent call last):
  File "D:\Anaconda3\envs\monai\lib\site-packages\monai\transforms\transform.py", line 82, in apply_transform
    return _apply_transform(transform, data, unpack_items)
  File "D:\Anaconda3\envs\monai\lib\site-packages\monai\transforms\transform.py", line 53, in _apply_transform
    return transform(parameters)
  File ".\testimg.py", line 35, in __call__
    d[key] = self.convertIMG(d[key].squeeze(0))
  File ".\testimg.py", line 20, in convertIMG  
    graph.clear()   #There is a problem here...
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axes\_base.py", line 1475, in clear
    self.cla()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axes\_base.py", line 1199, in cla
    spine.clear()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\spines.py", line 223, in clear
    self.axis.clear()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 818, in clear
    self.reset_ticks()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 846, in reset_ticks
    self.set_clip_path(self.axes.patch)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 936, in set_clip_path
    for child in self.majorTicks + self.minorTicks:
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 592, in __get__
    tick = instance._get_tick(major=True)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 2057, in _get_tick
    return XTick(self.axes, 0, major=major, **tick_kw)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 419, in __init__
    data=([0], [0]), transform=ax.get_xaxis_transform("tick1"))
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axes\_base.py", line 856, in get_xaxis_transform
    return self.spines.bottom.get_spine_transform()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\spines.py", line 332, in get_spine_transform
    self._ensure_position_is_set()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\spines.py", line 204, in _ensure_position_is_set
    self.set_position(self._position)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\spines.py", line 322, in set_position
    self.axis.reset_ticks()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 846, in reset_ticks
    self.set_clip_path(self.axes.patch)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 936, in set_clip_path
    for child in self.majorTicks + self.minorTicks:
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 592, in __get__
    tick = instance._get_tick(major=True)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 2057, in _get_tick
    return XTick(self.axes, 0, major=major, **tick_kw)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 419, in __init__
    data=([0], [0]), transform=ax.get_xaxis_transform("tick1"))
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axes\_base.py", line 856, in get_xaxis_transform
    return self.spines.bottom.get_spine_transform()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\spines.py", line 332, in get_spine_transform
    self._ensure_position_is_set()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\spines.py", line 204, in _ensure_position_is_set
    self.set_position(self._position)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\spines.py", line 322, in set_position
    self.axis.reset_ticks()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 846, in reset_ticks
    self.set_clip_path(self.axes.patch)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 936, in set_clip_path
    for child in self.majorTicks + self.minorTicks:
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 597, in __get__
    tick = instance._get_tick(major=False)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 2057, in _get_tick
    return XTick(self.axes, 0, major=major, **tick_kw)
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axis.py", line 421, in __init__
    data=([0], [1]), transform=ax.get_xaxis_transform("tick2"))
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\axes\_base.py", line 859, in get_xaxis_transform
    return self.spines.top.get_spine_transform()
  File "D:\Anaconda3\envs\monai\lib\site-packages\matplotlib\spines.py", line 340, in get_spine_transform
    assert len(position) == 2, 'position should be 2-tuple'
TypeError: object of type 'NoneType' has no len()

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File ".\testimg.py", line 42, in <module>
    transform=Compose(convertImage(keys=['img']))),
  File "D:\Anaconda3\envs\monai\lib\site-packages\monai\data\dataset.py", line 607, in __init__
    self._cache: List = self._fill_cache()
  File "D:\Anaconda3\envs\monai\lib\site-packages\monai\data\dataset.py", line 632, in _fill_cache
    desc="Loading dataset",
  File "D:\Anaconda3\envs\monai\lib\site-packages\tqdm\std.py", line 1180, in __iter__
    for obj in iterable:
  File "D:\Anaconda3\envs\monai\lib\multiprocessing\pool.py", line 748, in next
    raise value
  File "D:\Anaconda3\envs\monai\lib\multiprocessing\pool.py", line 121, in worker
    result = (True, func(*args, **kwds))
  File "D:\Anaconda3\envs\monai\lib\site-packages\monai\data\dataset.py", line 648, in _load_cache_item
    item = apply_transform(_xform, item)
  File "D:\Anaconda3\envs\monai\lib\site-packages\monai\transforms\transform.py", line 106, in apply_transform
    raise RuntimeError(f"applying transform {transform}") from e
RuntimeError: applying transform <__main__.convertImage object at 0x0000024C88D4DB08>

[yiheng-wang-nv]

Hi @Minxiangliu , I tried your code and it seems there is no such error on my side:

Could you provide the environment you used? You can add the following two lines into your code:

from monai.config import print_config
print_config()

[Minxiangliu]

@yiheng-wang-nv Thanks for your test!
My Environment:

print_config()

MONAI version: 0.8.0
Numpy version: 1.21.5
Pytorch version: 1.11.0+cu113
MONAI flags: HAS_EXT = False, USE_COMPILED = False
MONAI rev id: 714d00dffe6653e21260160666c4c201ab66511b

Optional dependencies:
Pytorch Ignite version: 0.4.8
Nibabel version: 3.2.1
scikit-image version: 0.19.1
Pillow version: 9.0.0
Tensorboard version: 2.8.0
gdown version: 4.2.0
TorchVision version: 0.12.0+cu113
tqdm version: 4.62.3
lmdb version: 1.3.0
psutil version: 5.9.0
pandas version: 1.3.5
einops version: 0.4.0
transformers version: 4.16.0
mlflow version: 1.23.1

pip lis

absl-py                   1.0.0
aiohttp                   3.8.1
aiosignal                 1.2.0
alembic                   1.7.5
argon2-cffi               21.3.0
argon2-cffi-bindings      21.2.0
async-timeout             4.0.2
asynctest                 0.13.0
attrs                     21.4.0
backcall                  0.2.0
beautifulsoup4            4.10.0
bleach                    4.1.0
cachetools                5.0.0
certifi                   2021.10.8
cffi                      1.15.0
charset-normalizer        2.0.10
click                     8.0.3
cloudpickle               2.0.0
colorama                  0.4.4
cycler                    0.11.0
databricks-cli            0.16.2
debugpy                   1.5.1
decorator                 5.1.1
defusedxml                0.7.1
docker                    5.0.3
einops                    0.4.0
entrypoints               0.3
filelock                  3.4.2
Flask                     2.0.2
fonttools                 4.30.0
frozenlist                1.3.0
fsspec                    2022.1.0
future                    0.18.2
gdown                     4.2.0
get-rolling-window        1.0.0
gitdb                     4.0.9
GitPython                 3.1.26
google-auth               2.5.0
google-auth-oauthlib      0.4.6
greenlet                  1.1.2
grpcio                    1.43.0
huggingface-hub           0.4.0
idna                      3.3
imageio                   2.14.1
importlib-metadata        4.10.1
importlib-resources       5.4.0
ipykernel                 6.9.2
ipython                   7.32.0
ipython-genutils          0.2.0
ipywidgets                7.6.5
itsdangerous              2.0.1
jedi                      0.18.1
Jinja2                    3.0.3
joblib                    1.1.0
jsonschema                4.4.0
jupyter-client            7.1.2
jupyter-core              4.9.2
jupyterlab-pygments       0.1.2
jupyterlab-widgets        1.0.2
kiwisolver                1.4.0
lmdb                      1.3.0
Mako                      1.1.6
Markdown                  3.3.6
MarkupSafe                2.0.1
matplotlib                3.5.1
matplotlib-inline         0.1.3
mistune                   0.8.4
mlflow                    1.23.1
monai                     0.8.0
multidict                 6.0.2
nbclient                  0.5.13
nbconvert                 6.4.4
nbformat                  5.2.0
nest-asyncio              1.5.4
networkx                  2.6.3
nibabel                   3.2.1
notebook                  6.4.9
numpy                     1.21.5
oauthlib                  3.1.1
opencv-python             4.5.5.64
packaging                 21.3
pandas                    1.3.5
pandocfilters             1.5.0
parso                     0.8.3
pickleshare               0.7.5
Pillow                    9.0.0
pip                       21.2.4
prometheus-client         0.13.0
prometheus-flask-exporter 0.18.7
prompt-toolkit            3.0.28
protobuf                  3.19.3
psutil                    5.9.0
pyasn1                    0.4.8
pyasn1-modules            0.2.8
pycparser                 2.21
pyDeprecate               0.3.1
pydicom                   2.2.2
Pygments                  2.11.2
pylibjpeg                 1.4.0
pyparsing                 3.0.7
pyrsistent                0.18.1
PySocks                   1.7.1
python-dateutil           2.8.2
python-gdcm               3.0.10.2
pytorch-ignite            0.4.8
pytorch-lightning         1.5.9
pytz                      2021.3
PyWavelets                1.2.0
pywin32                   227
pywinpty                  2.0.5
PyYAML                    6.0
pyzmq                     22.3.0
querystring-parser        1.2.4
regex                     2022.1.18
requests                  2.27.1
requests-oauthlib         1.3.0
rsa                       4.8
sacremoses                0.0.47
scikit-image              0.19.1
scipy                     1.7.3
Send2Trash                1.8.0
setuptools                59.5.0
six                       1.16.0
smmap                     5.0.0
soupsieve                 2.3.1
SQLAlchemy                1.4.31
sqlparse                  0.4.2
tabulate                  0.8.9
tensorboard               2.8.0
tensorboard-data-server   0.6.1
tensorboard-plugin-wit    1.8.1
terminado                 0.13.3
testpath                  0.6.0
tifffile                  2021.11.2
tokenizers                0.11.4
torch                     1.11.0+cu113
torchaudio                0.11.0+cu113
torchmetrics              0.7.0
torchvision               0.12.0+cu113
tornado                   6.1
tqdm                      4.62.3
traitlets                 5.1.1
transformers              4.16.0
typing_extensions         4.0.1
urllib3                   1.26.8
waitress                  2.0.0
wcwidth                   0.2.5
webencodings              0.5.1
websocket-client          1.2.3
Werkzeug                  2.0.2
wheel                     0.37.1
widgetsnbextension        3.5.2
wincertstore              0.2
yarl                      1.7.2
zipp                      3.7.0

[yiheng-wang-nv]

Hi @Minxiangliu , could you set the argument num_workers into the CacheDataset? It seems num_workers = None raise this issue. In the lastest commit which I used, the default value of it is changed from None to 1 thus no error happen.

Hi @Nic-Ma and @wyli , set num_workers = None for CacheDataset still has the issue, we may need to figure out the root cause and fix it.

[Minxiangliu]

Hi @yiheng-wang-nv,
Thank you very much for your help, I can make the code work properly by modifying num_workers.
In addition, I'm not sure if it's all num_workers problems. In the description of matplotlib, Threads may cause some problems. This is for your reference, thank you!