Compute metrics every n batches

[Ch-rode]

Hello ! I'm trying to use Pytorch lightning but I don't have clear in my mind all the steps. Anyway I'm trying to calculate the train_loss (for example) not only for each step(=batch) or epochs but every n batches (i.e. 500) but I'm not sure how to compute it (compute, reset etc). I tried this approach but this is not working. Can you help me? thanks

Any other suggestion on the code is well appreciated !

def training_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict:
        """ 
        Runs one training step. This usually consists in the forward function followed
            by the loss function.
        
        :param batch: The output of your dataloader. 
        :param batch_nb: Integer displaying which batch this is
        Returns:
            - dictionary containing the loss and the metrics to be added to the lightning logger.
        """
        inputs, targets = batch
        model_out = self.forward(**inputs)
        loss_val = self.loss(model_out, targets)
        


        y = targets["labels"]
        y_hat = model_out["logits"]
        
        labels_hat = torch.argmax(y_hat, dim=1)
        val_acc = self.metric_acc(labels_hat, y)
        tqdm_dict = {"train_loss": loss_val, 'batch_nb': batch_nb}
        self.log('train_loss', loss_val, on_step=True, on_epoch=True,prog_bar=True)
        self.log('train_acc', val_acc, on_step=True, prog_bar=True,on_epoch=True)

        # reset the metric to restart accumulating
        self.loss_val_bn = self.loss(model_out, targets) #accumulate state
        if batch_nb % 500 == 0:
            self.log("x batches test loss_train", self.loss_val_bn.compute(),batch_nb) # perform a compute every 10 batches
            self.loss_val_bn.reset() 


        #output = OrderedDict(
            #{"loss": loss_val, "progress_bar": tqdm_dict, "log": tqdm_dict})

        # can also return just a scalar instead of a dict (return loss_val)
        #return output
        return loss_val

[rohitgr7]

hey @Ch-rode !

what is self.loss here?
asking because you are calling it twice here:

loss_val = self.loss(model_out, targets)
...
self.loss_val_bn = self.loss(model_out, targets) #accumulate state

is it a torchmetric instance?

[Ch-rode]

Thank you for your answer.
yes i'm calling double only to check if with .compute and .reset the result will be different.
The loss is defined as following:

def __build_loss(self):
        """ Initializes the loss function/s. """
        self._loss = nn.CrossEntropyLoss()

def loss(self, predictions: dict, targets: dict) -> torch.tensor:
        """
        Computes Loss value according to a loss function.
        :param predictions: model specific output. Must contain a key 'logits' with
            a tensor [batch_size x 1] with model predictions
        :param labels: Label values [batch_size]
        Returns:
            torch.tensor with loss value.
        """
        return self._loss(predictions["logits"], targets["labels"])`

[Ch-rode]

Can I kindly ask you to share your code to have such output (for the training_step/valifation_step/validation_epoch_end)? I tried what you did but for sure i'm doing something wrong because nothuings changes, espcially I have only 1 row for epoch 0. MY pytorch lightning version is 1.6.0rc0

[rohitgr7]

here, it's very basic:

import torch
from torch.utils.data import DataLoader, Dataset

from pytorch_lightning import LightningModule, Trainer
from pytorch_lightning.loggers import CSVLogger


class RandomDataset(Dataset):
    def __init__(self, size, length):
        self.len = length
        self.data = torch.randn(length, size)

    def __getitem__(self, index):
        return self.data[index]

    def __len__(self):
        return self.len


class AverageMeter(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.count = 0

    def update(self, val):
        self.val += val
        self.count += 1

    def compute(self):
        return self.val / self.count


class BoringModel(LightningModule):
    def __init__(self):
        super().__init__()
        self.layer = torch.nn.Linear(32, 2)
        self.loss_meter = AverageMeter()

    def forward(self, x):
        return self.layer(x)

    def training_step(self, batch, batch_idx):
        loss = self(batch).sum()
        self.log("train_loss", loss)
        self.loss_meter.update(loss)
        if self.global_step % 1000 == 0:
            self.log('loss_avg_over_batches', self.loss_meter.compute())
            self.loss_meter.reset()
        return {"loss": loss}

    def validation_step(self, batch, batch_idx):
        loss = self(batch).sum()
        self.log("valid_loss", loss)

    def test_step(self, batch, batch_idx):
        loss = self(batch).sum()
        self.log("test_loss", loss)

    def configure_optimizers(self):
        return torch.optim.SGD(self.layer.parameters(), lr=0.1)


def run():
    train_data = DataLoader(RandomDataset(32, 6400), batch_size=2)
    val_data = DataLoader(RandomDataset(32, 6400), batch_size=2)
    logger = CSVLogger(save_dir='.')

    model = BoringModel()
    trainer = Trainer(
        default_root_dir=os.getcwd(),
        logger=logger,
        num_sanity_val_steps=0,
        max_epochs=5,
        enable_model_summary=False,
        log_every_n_steps=1000,
    )
    trainer.fit(model, train_dataloaders=train_data, val_dataloaders=val_data)


if __name__ == "__main__":
    run()

[Ch-rode]

Indeed, I also re-edit again my code as yours but still not printing nothing after 1000 steps. I also removed the validation_epoch_end as I read on the forum because now it is not needed anymore if you use self.log.

def training_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict:

        inputs, targets = batch
        model_out = self.forward(**inputs)
        loss_val = self.loss(model_out, targets)
        self.log("train_loss", loss_val)

        self.loss_meter.update(loss_val)
        if self.global_step % 1000 == 0:
            self.log('loss_avg_over_batches', self.loss_meter.compute())
        
        return {"loss": loss_val}

def validation_step(self, batch: tuple, batch_nb: int, *args, **kwargs) -> dict:

        inputs, targets = batch

        model_out = self.forward(**inputs)
        loss_val = self.loss(model_out, targets)


        y = targets["labels"]
        y_hat = model_out["logits"]
        
        labels_hat = torch.argmax(y_hat, dim=1)
        val_acc = self.metric_acc(labels_hat, y)

        self.log("valid_loss", loss_val)
        self.log("valid_acc", val_acc)
        

trainer = Trainer(
    log_every_n_steps=1000,
    gpus=hparams.gpus,
    tpu_cores=hparams.tpu_cores,
    logger=logger,
    callbacks=early_stop_callback,
    strategy="dp",
    max_epochs=hparams.max_epochs,
    min_epochs=hparams.min_epochs,
    accumulate_grad_batches=hparams.accumulate_grad_batches,
    limit_val_batches=hparams.val_percent_check,
    checkpoint_callback = checkpoint_callback,
    precision=hparams.precision,
    amp_level=hparams.amp_level,
    deterministic=True,
    amp_backend="apex",
    auto_lr_find=False,     
    auto_scale_batch_size=False,
    enable_model_summary=False,
    #progress_bar_refresh_rate=50
)

Really sorry to bother you but I'm trying to understand !

[rohitgr7]

can you try running my code shared above and see if it works for you?
if it does, then I'd probably need a small reproducible example to check the issue on your side. It's not that hard :)

[Ch-rode]

I just tried your code as it is and it works perfectly, as you shared metrics.csv with me
i do not understand which can be the problem