drop unused test with result api (#5058)

Co-authored-by: chaton <[email protected]>
Co-authored-by: Rohit Gupta <[email protected]>

Author: 3 people

tests/base/deterministic_model.py

@@ -15,7 +15,6 @@
 from torch import nn
 from torch.utils.data import Dataset, DataLoader
 
-from pytorch_lightning.core.step_result import TrainResult, EvalResult
 from pytorch_lightning.core.lightning import LightningModule
 
 
@@ -111,235 +110,6 @@ def training_epoch_end_scalar(self, outputs):
                 assert batch_out.grad_fn is None
                 assert isinstance(batch_out, torch.Tensor)
 
-    def training_step_no_default_callbacks_for_train_loop(self, batch, batch_idx):
-        """
-        Early stop and checkpoint only on these values
-        """
-        acc = self.step(batch, batch_idx)
-        result = TrainResult(minimize=acc)
-        assert 'early_step_on' not in result
-        assert 'checkpoint_on' in result
-        return result
-
-    def training_step_no_callbacks_result_obj(self, batch, batch_idx):
-        """
-        Early stop and checkpoint only on these values
-        """
-        acc = self.step(batch, batch_idx)
-        result = TrainResult(minimize=acc, checkpoint_on=False)
-        assert 'early_step_on' not in result
-        assert 'checkpoint_on' not in result
-        return result
-
-    def training_step_result_log_epoch_and_step_for_callbacks(self, batch, batch_idx):
-        """
-        Early stop and checkpoint only on these values
-        """
-        acc = self.step(batch, batch_idx)
-
-        self.assert_backward = False
-        losses = [20, 19, 18, 10, 15, 14, 9, 11, 11, 20]
-        idx = self.current_epoch
-        loss = acc + losses[idx]
-        result = TrainResult(minimize=loss, early_stop_on=loss, checkpoint_on=loss)
-        return result
-
-    def training_step_result_log_step_only(self, batch, batch_idx):
-        acc = self.step(batch, batch_idx)
-        result = TrainResult(minimize=acc)
-
-        # step only metrics
-        result.log(f'step_log_and_pbar_acc1_b{batch_idx}', torch.tensor(11).type_as(acc), prog_bar=True)
-        result.log(f'step_log_acc2_b{batch_idx}', torch.tensor(12).type_as(acc))
-        result.log(f'step_pbar_acc3_b{batch_idx}', torch.tensor(13).type_as(acc), logger=False, prog_bar=True)
-
-        self.training_step_called = True
-        return result
-
-    def training_step_result_log_epoch_only(self, batch, batch_idx):
-        acc = self.step(batch, batch_idx)
-        result = TrainResult(minimize=acc)
-
-        result.log(f'epoch_log_and_pbar_acc1_e{self.current_epoch}', torch.tensor(14).type_as(acc),
-                   on_epoch=True, prog_bar=True, on_step=False)
-        result.log(f'epoch_log_acc2_e{self.current_epoch}', torch.tensor(15).type_as(acc),
-                   on_epoch=True, on_step=False)
-        result.log(f'epoch_pbar_acc3_e{self.current_epoch}', torch.tensor(16).type_as(acc),
-                   on_epoch=True, logger=False, prog_bar=True, on_step=False)
-
-        self.training_step_called = True
-        return result
-
-    def training_step_result_log_epoch_and_step(self, batch, batch_idx):
-        acc = self.step(batch, batch_idx)
-        result = TrainResult(minimize=acc)
-
-        val_1 = (5 + batch_idx) * (self.current_epoch + 1)
-        val_2 = (6 + batch_idx) * (self.current_epoch + 1)
-        val_3 = (7 + batch_idx) * (self.current_epoch + 1)
-        result.log('step_epoch_log_and_pbar_acc1', torch.tensor(val_1).type_as(acc),
-                   on_epoch=True, prog_bar=True)
-        result.log('step_epoch_log_acc2', torch.tensor(val_2).type_as(acc),
-                   on_epoch=True)
-        result.log('step_epoch_pbar_acc3', torch.tensor(val_3).type_as(acc),
-                   on_epoch=True, logger=False, prog_bar=True)
-
-        self.training_step_called = True
-        return result
-
-    def training_epoch_end_return_for_log_epoch_and_step(self, result):
-        """
-        There should be an array of scalars without graphs that are all 171 (4 of them)
-        """
-        self.training_epoch_end_called = True
-
-        if self.use_dp or self.use_ddp2:
-            pass
-        else:
-            # only saw 4 batches
-            assert isinstance(result, TrainResult)
-
-        result.step_epoch_log_acc2 = result.step_epoch_log_acc2_step.prod()
-        result.step_epoch_pbar_acc3 = result.step_epoch_pbar_acc3_step.prod()
-        result.step_epoch_log_and_pbar_acc1 = result.step_epoch_log_and_pbar_acc1_step.prod()
-        result.minimize = result.minimize.mean()
-        result.checkpoint_on = result.checkpoint_on.mean()
-
-        result.step_epoch_log_and_pbar_acc1_step = result.step_epoch_log_and_pbar_acc1_step.prod()
-        result.step_epoch_log_and_pbar_acc1_epoch = result.step_epoch_log_and_pbar_acc1_epoch.prod()
-        result.step_epoch_log_acc2_step = result.step_epoch_log_acc2_step.prod()
-        result.step_epoch_log_acc2_epoch = result.step_epoch_log_acc2_epoch.prod()
-        result.step_epoch_pbar_acc3_step = result.step_epoch_pbar_acc3_step.prod()
-        result.step_epoch_pbar_acc3_epoch = result.step_epoch_pbar_acc3_epoch.prod()
-        result.log('epoch_end_log_acc', torch.tensor(1212).type_as(result.step_epoch_log_acc2_epoch),
-                   logger=True, on_epoch=True)
-        result.log('epoch_end_pbar_acc', torch.tensor(1213).type_as(result.step_epoch_log_acc2_epoch),
-                   logger=False, prog_bar=True, on_epoch=True)
-        result.log('epoch_end_log_pbar_acc', torch.tensor(1214).type_as(result.step_epoch_log_acc2_epoch),
-                   logger=True, prog_bar=True, on_epoch=True)
-        return result
-
-    # --------------------------
-    # EvalResults
-    # --------------------------
-    def validation_step_result_callbacks(self, batch, batch_idx):
-        acc = self.step(batch, batch_idx)
-
-        self.assert_backward = False
-        losses = [20, 19, 20, 21, 22, 23]
-        idx = self.current_epoch
-        loss = acc + losses[idx]
-        result = EvalResult(early_stop_on=loss, checkpoint_on=loss)
-
-        self.validation_step_called = True
-        return result
-
-    def validation_step_result_no_callbacks(self, batch, batch_idx):
-        acc = self.step(batch, batch_idx)
-
-        self.assert_backward = False
-        losses = [20, 19, 20, 21, 22, 23, 50, 50, 50, 50, 50, 50]
-        idx = self.current_epoch
-        loss = acc + losses[idx]
-
-        result = EvalResult(checkpoint_on=loss)
-
-        self.validation_step_called = True
-        return result
-
-    def validation_step_result_only_epoch_metrics(self, batch, batch_idx):
-        """
-        Only track epoch level metrics
-        """
-        acc = self.step(batch, batch_idx)
-        result = EvalResult(checkpoint_on=acc, early_stop_on=acc)
-
-        # step only metrics
-        result.log('no_val_no_pbar', torch.tensor(11 + batch_idx).type_as(acc), prog_bar=False, logger=False)
-        result.log('val_step_log_acc', torch.tensor(11 + batch_idx).type_as(acc), prog_bar=False, logger=True)
-        result.log('val_step_log_pbar_acc', torch.tensor(12 + batch_idx).type_as(acc), prog_bar=True, logger=True)
-        result.log('val_step_pbar_acc', torch.tensor(13 + batch_idx).type_as(acc), prog_bar=True, logger=False)
-
-        self.validation_step_called = True
-        return result
-
-    def validation_step_result_only_step_metrics(self, batch, batch_idx):
-        """
-        Only track epoch level metrics
-        """
-        acc = self.step(batch, batch_idx)
-        result = EvalResult(checkpoint_on=acc, early_stop_on=acc)
-
-        # step only metrics
-        result.log('no_val_no_pbar', torch.tensor(11 + batch_idx).type_as(acc),
-                   prog_bar=False, logger=False, on_epoch=False, on_step=True)
-        result.log('val_step_log_acc', torch.tensor(11 + batch_idx).type_as(acc),
-                   prog_bar=False, logger=True, on_epoch=False, on_step=True)
-        result.log('val_step_log_pbar_acc', torch.tensor(12 + batch_idx).type_as(acc),
-                   prog_bar=True, logger=True, on_epoch=False, on_step=True)
-        result.log('val_step_pbar_acc', torch.tensor(13 + batch_idx).type_as(acc),
-                   prog_bar=True, logger=False, on_epoch=False, on_step=True)
-        result.log('val_step_batch_idx', torch.tensor(batch_idx).type_as(acc),
-                   prog_bar=True, logger=True, on_epoch=False, on_step=True)
-
-        self.validation_step_called = True
-        return result
-
-    def validation_step_result_epoch_step_metrics(self, batch, batch_idx):
-        """
-        Only track epoch level metrics
-        """
-        acc = self.step(batch, batch_idx)
-        result = EvalResult(checkpoint_on=acc, early_stop_on=acc)
-
-        # step only metrics
-        result.log('no_val_no_pbar', torch.tensor(11 + batch_idx).type_as(acc),
-                   prog_bar=False, logger=False, on_epoch=True, on_step=True)
-        result.log('val_step_log_acc', torch.tensor(11 + batch_idx).type_as(acc),
-                   prog_bar=False, logger=True, on_epoch=True, on_step=True)
-        result.log('val_step_log_pbar_acc', torch.tensor(12 + batch_idx).type_as(acc),
-                   prog_bar=True, logger=True, on_epoch=True, on_step=True)
-        result.log('val_step_pbar_acc', torch.tensor(13 + batch_idx).type_as(acc),
-                   prog_bar=True, logger=False, on_epoch=True, on_step=True)
-        result.log('val_step_batch_idx', torch.tensor(batch_idx).type_as(acc),
-                   prog_bar=True, logger=True, on_epoch=True, on_step=True)
-
-        self.validation_step_called = True
-        return result
-
-    def validation_step_for_epoch_end_result(self, batch, batch_idx):
-        """
-        EvalResult flows to epoch end (without step_end)
-        """
-        acc = self.step(batch, batch_idx)
-        result = EvalResult(checkpoint_on=acc, early_stop_on=acc)
-
-        # step only metrics
-        result.log('val_step_metric', torch.tensor(batch_idx).type_as(acc),
-                   prog_bar=True, logger=True, on_epoch=True, on_step=False)
-        result.log('batch_idx', torch.tensor(batch_idx).type_as(acc),
-                   prog_bar=True, logger=True, on_epoch=True, on_step=False)
-
-        self.validation_step_called = True
-        return result
-
-    def validation_epoch_end_result(self, result):
-        self.validation_epoch_end_called = True
-
-        if self.trainer.running_sanity_check:
-            assert len(result.batch_idx) == 2
-        else:
-            assert len(result.batch_idx) == self.trainer.limit_val_batches
-
-        expected_val = result.val_step_metric.sum() / len(result.batch_idx)
-        result.val_step_metric = result.val_step_metric.mean()
-        result.batch_idx = result.batch_idx.mean()
-        assert result.val_step_metric == expected_val
-
-        result.log('val_epoch_end_metric', torch.tensor(189).type_as(result.val_step_metric), prog_bar=True)
-
-        return result
-
     # --------------------------
     # dictionary returns
     # --------------------------

tests/base/model_test_steps.py

@@ -59,38 +59,6 @@ def test_step(self, batch, batch_idx, *args, **kwargs):
                                   'test_dic': {'test_loss_a': loss_test}})
             return output
 
-    def test_step_result_obj(self, batch, batch_idx, *args, **kwargs):
-        """
-        Default, baseline test_step
-        :param batch:
-        :return:
-        """
-        x, y = batch
-        x = x.view(x.size(0), -1)
-        y_hat = self(x)
-
-        loss_test = self.loss(y, y_hat)
-
-        # acc
-        labels_hat = torch.argmax(y_hat, dim=1)
-        test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
-        test_acc = torch.tensor(test_acc)
-
-        test_acc = test_acc.type_as(x)
-
-        result = EvalResult()
-        # alternate possible outputs to test
-        if batch_idx % 1 == 0:
-            result.log_dict({'test_loss': loss_test, 'test_acc': test_acc})
-            return result
-        if batch_idx % 2 == 0:
-            return test_acc
-
-        if batch_idx % 3 == 0:
-            result.log_dict({'test_loss': loss_test, 'test_acc': test_acc})
-            result.test_dic = {'test_loss_a': loss_test}
-            return result
-
     def test_step__multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs):
         """
         Default, baseline test_step

tests/base/model_train_steps.py

@@ -53,25 +53,6 @@ def training_step(self, batch, batch_idx, optimizer_idx=None):
         )
         return output
 
-    def training_step_result_obj(self, batch, batch_idx, optimizer_idx=None):
-        # forward pass
-        x, y = batch
-        x = x.view(x.size(0), -1)
-        y_hat = self(x)
-
-        # calculate loss
-        loss_val = self.loss(y, y_hat)
-        log_val = loss_val
-
-        # alternate between tensors and scalars for "log" and "progress_bar"
-        if batch_idx % 2 == 0:
-            log_val = log_val.item()
-
-        result = TrainResult(loss_val)
-        result.log('some_val', log_val * log_val, prog_bar=True, logger=False)
-        result.log('train_some_val', log_val * log_val)
-        return result
-
     def training_step__inf_loss(self, batch, batch_idx, optimizer_idx=None):
         output = self.training_step(batch, batch_idx, optimizer_idx)
         if batch_idx == self.test_step_inf_loss:
@@ -81,19 +62,6 @@ def training_step__inf_loss(self, batch, batch_idx, optimizer_idx=None):
                 output /= 0
         return output
 
-    def training_step_full_loop_result_obj_dp(self, batch, batch_idx, optimizer_idx=None):
-        """
-        Full loop flow train step (result obj + dp)
-        """
-        x, y = batch
-        x = x.view(x.size(0), -1)
-        y_hat = self(x.to(self.device))
-        loss_val = y_hat.sum()
-        result = TrainResult(minimize=loss_val)
-        result.log('train_step_metric', loss_val + 1)
-        self.training_step_called = True
-        return result
-
     def training_step_result_obj_dp(self, batch, batch_idx, optimizer_idx=None):
         # forward pass
         x, y = batch
@@ -136,23 +104,6 @@ def training_epoch_end_full_loop_result_obj_dp(self, result):
 
         return result
 
-    def eval_step_full_loop_result_obj_dp(self, batch, batch_idx, optimizer_idx=None):
-        """
-        Full loop flow train step (result obj + dp)
-        """
-        x, y = batch
-        x = x.view(x.size(0), -1)
-        y_hat = self(x.to(self.device))
-        loss_val = y_hat.sum()
-        result = EvalResult(checkpoint_on=loss_val, early_stop_on=loss_val)
-
-        eval_name = 'validation' if not self.trainer.testing else 'test'
-        result.log(f'{eval_name}_step_metric', loss_val + 1, on_step=True)
-
-        setattr(self, f'{eval_name}_step_called', True)
-
-        return result
-
     def eval_step_end_full_loop_result_obj_dp(self, result):
         """
         Full loop flow train step (result obj + dp)
@@ -198,20 +149,3 @@ def eval_epoch_end_full_loop_result_obj_dp(self, result):
         setattr(result, f'{eval_name}_step_metric', reduced)
 
         return result
-
-    def training_step__using_metrics(self, batch, batch_idx, optimizer_idx=None):
-        """Lightning calls this inside the training loop"""
-        # forward pass
-        x, y = batch
-        x = x.view(x.size(0), -1)
-        y_hat = self(x)
-
-        # calculate loss
-        loss_val = self.loss(y, y_hat)
-
-        # call metric
-        val = self.metric(x, y)
-
-        result = TrainResult(minimize=loss_val)
-        result.log('metric_val', val)
-        return result

tests/base/model_valid_steps.py

@@ -71,25 +71,6 @@ def validation_step_no_monitor(self, batch, batch_idx, *args, **kwargs):
         })
         return output
 
-    def validation_step_result_obj(self, batch, batch_idx, *args, **kwargs):
-        x, y = batch
-        x = x.view(x.size(0), -1)
-        y_hat = self(x)
-
-        loss_val = self.loss(y, y_hat)
-
-        # acc
-        labels_hat = torch.argmax(y_hat, dim=1)
-        val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
-        val_acc = torch.tensor(val_acc).type_as(x)
-
-        result = EvalResult(checkpoint_on=loss_val, early_stop_on=loss_val)
-        result.log_dict({
-            'val_loss': loss_val,
-            'val_acc': val_acc,
-        })
-        return result
-
     def validation_step_result_obj_dp(self, batch, batch_idx, *args, **kwargs):
         x, y = batch
         x = x.view(x.size(0), -1)