Only log the first test dataloader results.

[AristoYU]

Hello, I use multiple dataloaders to test. But in tensorboad, only the first test dataloader result will be logged.

I use the pl.LightningDataModule to create the test dataloader list:

class MNISTData(Data):
    
    def __init__(self, data_dir="./", batch_size=100, num_workers=0):
        
        super(MNISTData, self).__init__()
        self.num_workers = num_workers
        
        MNIST(data_dir, train=True, download=True)
        MNIST(data_dir, train=False, download=True)
        
        self.data_dir = data_dir
        self.batch_size = batch_size
        self.transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,))
        ])

    def setup(self, stage=None):

        # Assign train/val datasets for use in dataloaders
        if stage == 'fit' or stage is None:
            mnist_full = MNIST(self.data_dir, train=True, transform=self.transform)
            self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000])

        # Assign test dataset for use in dataloader(s)
        if stage == 'test' or stage is None:
            mnist_test = MNIST(self.data_dir, train=False, transform=self.transform)
            self.mnist_testA, self.mnist_testB = random_split(mnist_test, [5000, 5000])
            
    def train_dataloader(self):
        return DataLoader(self.mnist_train, self.batch_size, shuffle=True, 
                          num_workers=self.num_workers, pin_memory=True, collate_fn=collate_fn,)
    
    def val_dataloader(self):
        return DataLoader(self.mnist_val, self.batch_size, num_workers=self.num_workers, pin_memory=True, collate_fn=collate_fn,)
    
    def test_dataloader(self):
        return [
            DataLoader(self.mnist_val, self.batch_size, num_workers=self.num_workers, pin_memory=True, collate_fn=collate_fn,),
            DataLoader(self.mnist_testA, 1, num_workers=self.num_workers, pin_memory=True, collate_fn=collate_fn,),
            DataLoader(self.mnist_testB, 1, num_workers=self.num_workers, pin_memory=True, collate_fn=collate_fn,),
        ]

And I log the result in the test_step function:

pred = self.forward(**batch)
loss = self.model.loss(**pred, **batch)

prog_bar = (mode != "train")

for metric_name, metric in self.metrics.items():
      res = metric(pred["preds"], batch["target"])
      self.log(f"{metric_name}_{mode}", res.item(), prog_bar, sync_dist=True, add_dataloader_idx=True)

for key, value in loss.items():
      self.log(f"{key}_{mode}", value.item(), False, sync_dist=True)

After training, I got the result in the terminal:

But, when I open tensorboard, it cannot load the dataloader1 result:

I don't know what's wrong with my code.

[carmocca]

Can you check if trainer.logged_metrics contains both of the dataloader results?

If so, then it's probably an issue/bug with TensorBoard