[bugfix] Resolve metrics not being properly resetted on validation epoch end (#9717)

Co-authored-by: Carlos Mocholi <[email protected]>

Author: 2 people

CHANGELOG.md

@@ -9,9 +9,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Moved the gradient unscaling in `NativeMixedPrecisionPlugin` from `pre_optimizer_step` to `post_backward` ([#9606](https://github.com/PyTorchLightning/pytorch-lightning/pull/9606))
 - Fixed gradient unscaling being called too late, causing gradient clipping and gradient norm tracking to be applied incorrectly ([#9606](https://github.com/PyTorchLightning/pytorch-lightning/pull/9606))
-
-
 - Fixed `lr_find` to generate same results on multiple calls ([#9704](https://github.com/PyTorchLightning/pytorch-lightning/pull/9704))
+- Fixed `reset` metrics on validation epoch end ([#9717](https://github.com/PyTorchLightning/pytorch-lightning/pull/9717))
+
 
 
 ## [1.4.8] - 2021-09-22

pytorch_lightning/loops/dataloader/evaluation_loop.py

@@ -208,8 +208,8 @@ def on_evaluation_end(self, *args: Any, **kwargs: Any) -> None:
             # summarize profile results
             self.trainer.profiler.describe()
 
-        # reset any `torchmetrics.Metric` and the logger connector state
-        self.trainer.logger_connector.reset_results(metrics=True)
+        # reset the logger connector state
+        self.trainer.logger_connector.reset_results()
 
     def on_evaluation_epoch_start(self, *args: Any, **kwargs: Any) -> None:
         """Runs ``on_epoch_start`` and ``on_{validation/test}_epoch_start`` hooks"""

pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py

@@ -282,9 +282,9 @@ def reset_metrics(self) -> None:
         self._logged_metrics = {}
         self._callback_metrics = {}
 
-    def reset_results(self, metrics: Optional[bool] = None) -> None:
+    def reset_results(self) -> None:
         if self.trainer._results is not None:
-            self.trainer._results.reset(metrics=metrics)
+            self.trainer._results.reset()
 
         self._batch_idx = None
         self._split_idx = None

tests/trainer/logging_/test_eval_loop_logging.py

@@ -618,3 +618,59 @@ def validation_step(self, batch, batch_idx):
     )
 
     trainer.fit(model)
+
+
+@pytest.mark.parametrize("val_check_interval", [0.5, 1.0])
+def test_multiple_dataloaders_reset(val_check_interval, tmpdir):
+    class TestModel(BoringModel):
+        def training_step(self, batch, batch_idx):
+            out = super().training_step(batch, batch_idx)
+            value = 1 + batch_idx
+            if self.current_epoch != 0:
+                value *= 10
+            self.log("batch_idx", value, on_step=True, on_epoch=True, prog_bar=True)
+            return out
+
+        def training_epoch_end(self, outputs):
+            metrics = self.trainer.progress_bar_metrics
+            v = 15 if self.current_epoch == 0 else 150
+            assert metrics["batch_idx_epoch"] == (v / 5.0)
+
+        def validation_step(self, batch, batch_idx, dataloader_idx):
+            value = (1 + batch_idx) * (1 + dataloader_idx)
+            if self.current_epoch != 0:
+                value *= 10
+            self.log("val_loss", value, on_step=False, on_epoch=True, prog_bar=True, logger=True)
+            return value
+
+        def validation_epoch_end(self, outputs):
+            if self.current_epoch == 0:
+                assert sum(outputs[0]) / 5 == 3
+                assert sum(outputs[1]) / 5 == 6
+            else:
+                assert sum(outputs[0]) / 5 == 30
+                assert sum(outputs[1]) / 5 == 60
+
+            tot_loss = torch.mean(torch.tensor(outputs, dtype=torch.float))
+            if self.current_epoch == 0:
+                assert tot_loss == (3 + 6) / 2
+            else:
+                assert tot_loss == (30 + 60) / 2
+            assert self.trainer._results["validation_step.val_loss.0"].cumulated_batch_size == 5
+            assert self.trainer._results["validation_step.val_loss.1"].cumulated_batch_size == 5
+
+        def val_dataloader(self):
+            return [super().val_dataloader(), super().val_dataloader()]
+
+    model = TestModel()
+    trainer = Trainer(
+        default_root_dir=tmpdir,
+        limit_train_batches=5,
+        limit_val_batches=5,
+        num_sanity_val_steps=0,
+        val_check_interval=val_check_interval,
+        max_epochs=3,
+        log_every_n_steps=1,
+        weights_summary=None,
+    )
+    trainer.fit(model)