Add wall-clock val_check_interval with epoch alignment and timer reset

Author: Sohaib-Ahmed21

src/lightning/pytorch/loops/evaluation_loop.py

@@ -19,6 +19,7 @@
 from collections.abc import Iterable, Iterator
 from dataclasses import dataclass
 from typing import Any, Optional, Union
+import time
 
 from lightning_utilities.core.apply_func import apply_to_collection
 from torch import Tensor
@@ -313,6 +314,9 @@ def on_run_end(self) -> list[_OUT_DICT]:
 
         if self.verbose and self.trainer.is_global_zero:
             self._print_results(logged_outputs, self._stage.value)
+        
+        now = time.monotonic()
+        self.trainer._last_val_time = now
 
         return logged_outputs
 

src/lightning/pytorch/loops/fit_loop.py

@@ -14,6 +14,7 @@
 import logging
 from dataclasses import dataclass
 from typing import Any, Optional, Union
+import time
 
 import torch
 from typing_extensions import override
@@ -283,7 +284,13 @@ def setup_data(self) -> None:
         # store epoch of dataloader reset for reload_dataloaders_every_n_epochs
         self._last_train_dl_reload_epoch = trainer.current_epoch
 
-        if isinstance(trainer.val_check_interval, int):
+        # If time-based validation is enabled, disable batch-based scheduling here.
+        # Use None to clearly signal "no batch-based validation"; wall-time logic will run elsewhere.
+        if getattr(trainer, "_val_check_time_interval", None) is not None:
+            trainer.val_check_batch = None
+            trainer._train_start_time = time.monotonic()
+            trainer._last_val_time   = trainer._train_start_time
+        elif isinstance(trainer.val_check_interval, int):
             trainer.val_check_batch = trainer.val_check_interval
             if trainer.val_check_batch > self.max_batches and trainer.check_val_every_n_epoch is not None:
                 raise ValueError(
@@ -299,7 +306,7 @@ def setup_data(self) -> None:
                 else:
                     raise MisconfigurationException(
                         "When using an IterableDataset for `train_dataloader`,"
-                        " `Trainer(val_check_interval)` must be `1.0` or an int. An int k specifies"
+                        " `Trainer(val_check_interval)` must be time based, `1.0` or an int. An int k specifies"
                         " checking validation every k training batches."
                     )
             else:

src/lightning/pytorch/loops/training_epoch_loop.py

@@ -16,6 +16,7 @@
 from collections import OrderedDict
 from dataclasses import dataclass
 from typing import Any, Optional, Union
+import time
 
 import torch
 from typing_extensions import override
@@ -534,6 +535,13 @@ def _should_check_val_fx(self, data_fetcher: _DataFetcher) -> bool:
             # and when the loop allows to stop (min_epochs/steps met)
             return True
 
+        interval = self.trainer._val_check_time_interval
+        if interval is not None:
+            now = time.monotonic()
+            if now - self.trainer._last_val_time >= interval:
+                # time’s up → tell Trainer to validate
+                return True
+            return False
         # TODO: let training/eval loop handle logic around limit_*_batches and val_check_batch
         is_val_check_batch = is_last_batch
         if isinstance(self.trainer.limit_train_batches, int) and is_infinite_dataset:

src/lightning/pytorch/trainer/connectors/data_connector.py

@@ -15,6 +15,7 @@
 from collections.abc import Iterable
 from dataclasses import dataclass, field
 from typing import Any, Optional, Union
+from datetime import timedelta
 
 import torch.multiprocessing as mp
 from torch.utils.data import BatchSampler, DataLoader, RandomSampler, Sampler, SequentialSampler
@@ -50,7 +51,7 @@ def __init__(self, trainer: "pl.Trainer"):
 
     def on_trainer_init(
         self,
-        val_check_interval: Optional[Union[int, float]],
+        val_check_interval: Optional[Union[int, float, str, timedelta, dict]],
         reload_dataloaders_every_n_epochs: int,
         check_val_every_n_epoch: Optional[int],
     ) -> None:
@@ -63,8 +64,8 @@ def on_trainer_init(
 
         if check_val_every_n_epoch is None and isinstance(val_check_interval, float):
             raise MisconfigurationException(
-                "`val_check_interval` should be an integer when `check_val_every_n_epoch=None`,"
-                f" found {val_check_interval!r}."
+                "`val_check_interval` should be an integer or a time-based duration (str 'DD:HH:MM:SS', "
+                "datetime.timedelta, or dict kwargs for timedelta) when `check_val_every_n_epoch=None`."
             )
 
         self.trainer.check_val_every_n_epoch = check_val_every_n_epoch

src/lightning/pytorch/trainer/setup.py

@@ -14,6 +14,7 @@
 """Houses the methods used to set up the Trainer."""
 
 from typing import Optional, Union
+from datetime import timedelta
 
 import lightning.pytorch as pl
 from lightning.fabric.utilities.warnings import PossibleUserWarning
@@ -40,7 +41,7 @@ def _init_debugging_flags(
     limit_predict_batches: Optional[Union[int, float]],
     fast_dev_run: Union[int, bool],
     overfit_batches: Union[int, float],
-    val_check_interval: Optional[Union[int, float]],
+    val_check_interval: Optional[Union[int, float, str, timedelta, dict]],
     num_sanity_val_steps: int,
 ) -> None:
     # init debugging flags
@@ -69,6 +70,7 @@ def _init_debugging_flags(
         trainer.num_sanity_val_steps = 0
         trainer.fit_loop.max_epochs = 1
         trainer.val_check_interval = 1.0
+        trainer._val_check_time_interval = None  # time not applicable in fast_dev_run
         trainer.check_val_every_n_epoch = 1
         trainer.loggers = [DummyLogger()] if trainer.loggers else []
         rank_zero_info(
@@ -82,7 +84,16 @@ def _init_debugging_flags(
         trainer.limit_test_batches = _determine_batch_limits(limit_test_batches, "limit_test_batches")
         trainer.limit_predict_batches = _determine_batch_limits(limit_predict_batches, "limit_predict_batches")
         trainer.num_sanity_val_steps = float("inf") if num_sanity_val_steps == -1 else num_sanity_val_steps
-        trainer.val_check_interval = _determine_batch_limits(val_check_interval, "val_check_interval")
+        # Support time-based validation intervals:
+        # If `val_check_interval` is str/dict/timedelta, parse and store seconds on the trainer
+        # for the loops to consume.
+        trainer._val_check_time_interval = None  # default
+        if isinstance(val_check_interval, (str, dict, timedelta)):
+            trainer._val_check_time_interval = _parse_time_interval_seconds(val_check_interval)
+            # Keep the numeric scheduler neutral; loops should check the time-based attribute.
+            trainer.val_check_interval = 1.0
+        else:
+            trainer.val_check_interval = _determine_batch_limits(val_check_interval, "val_check_interval")
 
     if overfit_batches_enabled:
         trainer.limit_train_batches = overfit_batches
@@ -187,3 +198,30 @@ def _log_device_info(trainer: "pl.Trainer") -> None:
 
         if HPUAccelerator.is_available() and not isinstance(trainer.accelerator, HPUAccelerator):
             rank_zero_warn("HPU available but not used. You can set it by doing `Trainer(accelerator='hpu')`.")
+
+def _parse_time_interval_seconds(value: Union[str, timedelta, dict]) -> float:
+     if isinstance(value, timedelta):
+         return value.total_seconds()
+     if isinstance(value, dict):
+         td = timedelta(**value)
+         return td.total_seconds()
+     if isinstance(value, str):
+         parts = value.split(":")
+         if len(parts) != 4:
+             raise MisconfigurationException(
+                 f"Invalid time format for `val_check_interval`: {value!r}. Expected 'DD:HH:MM:SS'."
+             )
+         d, h, m, s = parts
+         try:
+             days = int(d)
+             hours = int(h)
+             minutes = int(m)
+             seconds = int(s)
+         except ValueError:
+             raise MisconfigurationException(
+                 f"Non-integer component in `val_check_interval` string: {value!r}. Use 'DD:HH:MM:SS'."
+             )
+         td = timedelta(days=days, hours=hours, minutes=minutes, seconds=seconds)
+         return td.total_seconds()
+     # Should not happen given the caller guards
+     raise MisconfigurationException(f"Unsupported type for `val_check_interval`: {type(value)!r}")

src/lightning/pytorch/trainer/trainer.py

@@ -109,7 +109,7 @@ def __init__(
         limit_test_batches: Optional[Union[int, float]] = None,
         limit_predict_batches: Optional[Union[int, float]] = None,
         overfit_batches: Union[int, float] = 0.0,
-        val_check_interval: Optional[Union[int, float]] = None,
+        val_check_interval: Optional[Union[int, float, str, timedelta, dict[str,int]]] = None,
         check_val_every_n_epoch: Optional[int] = 1,
         num_sanity_val_steps: Optional[int] = None,
         log_every_n_steps: Optional[int] = None,
@@ -203,12 +203,20 @@ def __init__(
                 after a fraction of the training epoch. Pass an ``int`` to check after a fixed number of training
                 batches. An ``int`` value can only be higher than the number of training batches when
                 ``check_val_every_n_epoch=None``, which validates after every ``N`` training batches
-                across epochs or during iteration-based training.
+                across epochs or during iteration-based training. Additionally, accepts a time-based duration
+                as a string "DD:HH:MM:SS", a :class:`datetime.timedelta`, or a dict of kwargs to
+                :class:`datetime.timedelta`. When time-based, validation triggers once the elapsed wall-clock time
+                since the last validation exceeds the interval; the check occurs after the current batch
+                completes, the validation loop runs, and the timer is reset.
                 Default: ``1.0``.
 
             check_val_every_n_epoch: Perform a validation loop after every `N` training epochs. If ``None``,
                 validation will be done solely based on the number of training batches, requiring ``val_check_interval``
-                to be an integer value.
+                to be an integer value. When used together with a time-based ``val_check_interval`` and 
+                ``check_val_every_n_epoch`` > 1, validation is aligned to epoch multiples: if the interval elapses
+                before the next multiple-N epoch, validation runs at the start of that epoch (after the first batch)
+                and the timer resets; if it elapses during a multiple-N epoch, validation runs after the current batch.
+                For ``None`` or ``1``, the time-based behavior of ``val_check_interval`` applies without additional alignment.
                 Default: ``1``.
 
             num_sanity_val_steps: Sanity check runs n validation batches before starting the training routine.