Setting num_sanity_val_steps = -1 Disables Sanity Check Instead of Running All Batches

[RaphaelLilt]

📚 Documentation

Description:
The documentation indicates that setting num_sanity_val_steps to -1 a sanity check is run on all validation batches. However, in Trainer._run_sanity_check, the sanity check only runs if self.num_sanity_val_steps > 0. As a result, when num_sanity_val_steps is set to -1, no sanity check is run.

Steps to Reproduce:

1. Instantiate Trainer with num_sanity_val_steps = -1.
2. Add validation dataloader.
3. Call trainer.fit(). Notice that no sanity check is run.

Expected Behavior:
When num_sanity_val_steps is -1, the sanity check should iterate over all validation batches.

Actual Behavior:
The condition (self.num_sanity_val_steps > 0) prevents any sanity checking when num_sanity_val_steps is -1.

Proposed Fix:
Update _run_sanity_check to treat -1 as a special value that means “run all validation batches”, or update the documentation.

[SkafteNicki]

Hi @RaphaelLilt, while it is true that in these lines

pytorch-lightning/src/lightning/pytorch/trainer/trainer.py

Lines 1071 to 1076 in 791753b

	should_sanity_check = (
	self.enable_validation
	and self.num_sanity_val_steps > 0
	# do not sanity check if restarting because it would mess up the loaded state
	and not val_loop.restarting
	)

it looks like sanity checking will not run when num_sanity_val_steps=-1, what is missing here is that the value gets converted to inf in this line

pytorch-lightning/src/lightning/pytorch/trainer/setup.py

Line 84 in 791753b

trainer.num_sanity_val_steps = float("inf") if num_sanity_val_steps == -1 else num_sanity_val_steps

before the check, which means that for num_sanity_val_steps=-1 works as intended. Converting to inf makes sure that in this line

pytorch-lightning/src/lightning/pytorch/trainer/trainer.py

Line 1568 in 791753b

return [min(self.num_sanity_val_steps, batches) for batches in max_batches]

the value is not restricting the number of batches evaluated.

As final evidence, I have attached a small testing script that checks that the behavior is correct:

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, TensorDataset
import lightning as L


class SimpleModel(L.LightningModule):
    """Simple model for testing sanity check behavior."""
    
    def __init__(self):
        super().__init__()
        self.layer = nn.Linear(10, 1)
        self.validation_step_calls = 0
        
    def forward(self, x):
        return self.layer(x)
    
    def training_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = F.mse_loss(y_hat.squeeze(), y)
        return loss
    
    def validation_step(self, batch, batch_idx):
        """Track validation step calls to verify sanity check behavior."""
        if self.trainer.sanity_checking:
            self.validation_step_calls += 1
            print(f"Validation step called: batch_idx={batch_idx}, total_calls={self.validation_step_calls}")
        
        x, y = batch
        y_hat = self(x)
        loss = F.mse_loss(y_hat.squeeze(), y)
        self.log('val_loss', loss)
        return loss
    
    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=0.001)


def create_dummy_data(num_samples=100, input_size=10):
    """Create dummy dataset for testing."""
    X = torch.randn(num_samples, input_size)
    y = torch.randn(num_samples)
    return TensorDataset(X, y)


def test_sanity_check_behavior():
    """Test the sanity check behavior with different num_sanity_val_steps values."""
    
    # Create datasets
    train_dataset = create_dummy_data(200, 10)
    val_dataset = create_dummy_data(50, 10)  # 5 batches with batch_size=10
    
    train_loader = DataLoader(train_dataset, batch_size=10, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size=10, shuffle=False)
    
    print("=" * 80)
    print("Testing sanity check behavior with different num_sanity_val_steps values")
    print("=" * 80)
    
    # Test case 1: num_sanity_val_steps = 2 (should run 2 validation steps)
    print("\n" + "="*50)
    print("TEST 1: num_sanity_val_steps = 2")
    print("Expected: Should run sanity check on 2 validation batches")
    print("="*50)
    
    model1 = SimpleModel()
    trainer1 = L.Trainer(
        max_epochs=1,
        num_sanity_val_steps=2,
        logger=False,
        enable_checkpointing=False,
        enable_progress_bar=False
    )
    
    print(f"Before fit: validation_step_calls = {model1.validation_step_calls}")
    trainer1.fit(model1, train_loader, val_loader)
    print(f"After fit: validation_step_calls = {model1.validation_step_calls}")
    
    # Test case 2: num_sanity_val_steps = -1 (should run all validation batches)
    print("\n" + "="*50)
    print("TEST 2: num_sanity_val_steps = -1")
    print("Expected: Should run sanity check on ALL validation batches (5 batches)")
    print("Actual bug: No sanity check is run at all")
    print("="*50)
    
    model2 = SimpleModel()
    trainer2 = L.Trainer(
        max_epochs=1,
        num_sanity_val_steps=-1,  # This should run sanity check on all batches
        logger=False,
        enable_checkpointing=False,
        enable_progress_bar=False
    )
    
    print(f"Before fit: validation_step_calls = {model2.validation_step_calls}")
    trainer2.fit(model2, train_loader, val_loader)
    print(f"After fit: validation_step_calls = {model2.validation_step_calls}")
    
    # Test case 3: num_sanity_val_steps = 0 (should disable sanity check)
    print("\n" + "="*50)
    print("TEST 3: num_sanity_val_steps = 0")
    print("Expected: Should disable sanity check completely")
    print("="*50)
    
    model3 = SimpleModel()
    trainer3 = L.Trainer(
        max_epochs=1,
        num_sanity_val_steps=0,
        logger=False,
        enable_checkpointing=False,
        enable_progress_bar=False
    )
    
    print(f"Before fit: validation_step_calls = {model3.validation_step_calls}")
    trainer3.fit(model3, train_loader, val_loader)
    print(f"After fit: validation_step_calls = {model3.validation_step_calls}")
    
    # Summary
    print("\n" + "="*80)
    print("SUMMARY")
    print("="*80)
    print(f"num_sanity_val_steps = 2:  validation_step_calls = {model1.validation_step_calls}")
    print(f"num_sanity_val_steps = -1: validation_step_calls = {model2.validation_step_calls}")
    print(f"num_sanity_val_steps = 0:  validation_step_calls = {model3.validation_step_calls}")
    
    print("\nBUG ANALYSIS:")
    if model2.validation_step_calls == 0:
        print("❌ BUG CONFIRMED: num_sanity_val_steps = -1 disables sanity check completely")
        print("   According to documentation, it should run sanity check on all validation batches")
    else:
        print("✅ Bug not reproduced - sanity check ran with num_sanity_val_steps = -1")
    
    print("\nEXPECTED BEHAVIOR:")
    print("- num_sanity_val_steps = 2:  Should run 2 validation steps (sanity check)")
    print("- num_sanity_val_steps = -1: Should run 5 validation steps (all batches)")
    print("- num_sanity_val_steps = 0:  Should run 0 validation steps (disabled)")


if __name__ == "__main__":
    # Set seed for reproducibility
    torch.manual_seed(42)
    
    test_sanity_check_behavior()

Closing issue, but feel free to ping and reopen if necessary.