How can i use multiple optimizers and lr_scheduler?

[morestart]

this is my code,

    def optimizer_step(
            self,
            epoch: int = None,
            batch_idx: int = None,
            optimizer: Optimizer = None,
            optimizer_idx: int = None,
            optimizer_closure: Optional[Callable] = None,
            on_tpu: bool = None,
            using_native_amp: bool = None,
            using_lbfgs: bool = None,
    ) -> None:
        if optimizer_idx == 0 and epoch < 100:
            optimizer.step(closure=optimizer_closure)
        elif optimizer_idx == 1 and epoch >= 100:
            optimizer.step(closure=optimizer_closure)

    def configure_optimizers(self):
        optimizer1 = optim.get_optimizer(self.parameters(), 'adam', self.lr)
        optimizer2 = optim.get_optimizer(self.parameters(), 'sgd', self.lr)

        optimizers = [optimizer1, optimizer2]
        lr_scheduler1 = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer1)
        lr_scheduler2 = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer2)

        return {"optimizer": optimizers, "lr_scheduler": lr_scheduler, "monitor": "val_loss"}
        # return optimizers, [lr_scheduler1, lr_scheduler2]

I want to use Adam in the first 100 epoch, and then use sgd, can I use a global learning rate scheduler to adjust the learning rate?

[rohitgr7]

I am afraid there is no clean way to do this since by default it will .. one lazy hack can be, just flip the optimizer & scheduler:

class LitModel(LightningModule):
    def on_train_epoch_start(self):
        if self.current_epoch == 100:
            self.trainer.accelerator.setup_optimizers(self.trainer)
    
    def configure_optimizers(self):
        if self.current_epoch == 100:
            optimizer = optim.get_optimizer(self.parameters(), 'sgd', self.lr)
            lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer)
            lr_scheduler.load_state_dict(self.trainer.lr_schedulers[0].state_dict())  ## only if you want to load the current state of the old learning rate.
        else:
            optimizer = optim.get_optimizer(self.parameters(), 'adam', self.lr)
            lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer)
        
        return {'optimizer': optimizer, 'lr_scheduler': lr_scheduler, "monitor": "val_loss"}
            

do you want the new optimizer to have recently update lr during init?

[morestart]

thanks for you answers!
i want the new optimizer to have the last recently update lr to continue train model...

[rohitgr7]

then you might need to restore the LR as well. you can use self.trainer.optimizers[0].state_dict() to get that.