How can run two diffrent model with own's optimizer and loss?

[blakeliu]

Flowing https://pytorch-lightning.readthedocs.io/en/latest/common/optimizers.html?highlight=configure_optimizers

My code as below:

class Landmarker(pl.LightningModule):
    def __init__(self, is_show=False):
        super(Landmarker, self).__init__()
        self._model = KeypointModel()
        self._discriminator = Discriminator()
        self.landmark_loss = LandmarkMultiLoss()
        self.gan_loss = HeatmapGANLoss()
        self.automatic_optimization = False

    def configure_optimizers(self):
        optimizer = eval("torch.optim." + cfg.SOLVER.OPTIMIZER)(self._model.parameters(),
                                                                **cfg.SOLVER.OPTIMIZER_PARAMETER)
        scheduler = eval("torch.optim.lr_scheduler." + cfg.SOLVER.SCHEDULER)(optimizer,
                                                                             **cfg.SOLVER.SCHEDULER_PARAMETER)
        base_optimizer = {"optimizer": optimizer, "lr_scheduler": scheduler, "monitor": "train_loss"}

        optimizer_estimator = eval("torch.optim." + cfg.SOLVER.OPTIMIZER)(self._discriminator.parameters(),
                                                                          **cfg.SOLVER.OPTIMIZER_PARAMETER)
        scheduler_estimator = eval("torch.optim.lr_scheduler." + cfg.SOLVER.SCHEDULER)(optimizer_estimator,
                                                                                       **cfg.SOLVER.SCHEDULER_PARAMETER)
        return base_optimizer,   {"optimizer": optimizer_estimator, "lr_scheduler": scheduler_estimator}


    def forward(self, x):
        out = self._model(x)
        discrim_prob = self._discriminator(out["heatmap"])
       return out, discrim_prob

    def training_step(self, batch, batch_idx, optimizer_idx):
        opt_base, opt_gan = self.optimizers()
        x = batch["img"]
        y_hat = self._model(x)

        loss = self.landmark_loss(y_hat, batch)
        opt_base.zero_grad()
       self.manual_backward(loss, opt_base)
       opt_base.step()

        opt_gan.zero_grad()
        discrim_prob = self._discriminator(y_hat['heatmap'])
        g_loss = self.gan_loss(y_hat['heatmap'], batch['heatmap'], discrim_prob)
        self.manual_backward(g_loss, opt_gan)
        opt_gan.step()
        self.log_dict({"train_loss": loss, "g_loss": g_loss}, on_step=False, prog_bar=True, on_epoch=True, logger=True)                                    

RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time.

@williamFalcon Please help me, thanks!

[kkirtac]

Would this help? #2783 (comment)

[blakeliu]

class Landmarker(pl.LightningModule):
    def __init__(self, is_show=False):
        super(Landmarker, self).__init__()
        self._model = KeypointModel()
        self._discriminator = Discriminator()
        self.landmark_loss = LandmarkMultiLoss()
        self.gan_loss = HeatmapGANLoss()
        self.automatic_optimization = True

...

    def training_step(self, batch, batch_idx, optimizer_idx):
        optimizers = self.optimizers()
        if optimizer_idx == 0:
            x = batch["img"]
            self.gt_hm = batch['heatmap']
            self.y_hat = self._model(x)
            self.discrim_prob = self._discriminator(self.y_hat['heatmap'])
            loss = self.landmark_loss(self.y_hat, batch)
            self.log("loss", loss, on_step=False, prog_bar=True, on_epoch=True, logger=True)
            return loss
        elif optimizer_idx == 1:
            g_loss = self.gan_loss(self.y_hat['heatmap'].detach(), self.gt_hm.detach(), self.discrim_prob)
            self.log("g_loss", g_loss, on_step=False, prog_bar=True, on_epoch=True, logger=True)
            return g_loss

still bug:

RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time.

[akihironitta]

@blakeliu Could you provide a full script, stack traces, and your environment info?

[awaelchli]

Have you tried to follow the instructions of the error message?

self.manual_backward(loss, opt_base, retain_graph=True)