update lr scheduler and precision

Author: aladdinpersson

ML/Pytorch/Basics/pytorch_lr_ratescheduler.py

@@ -3,13 +3,12 @@
 case with a (very) small and simple Feedforward Network training on MNIST
 dataset with a learning rate scheduler. In this case ReduceLROnPlateau
 scheduler is used, but can easily be changed to any of the other schedulers
-available.
-
-Video explanation: https://youtu.be/P31hB37g4Ak
-Got any questions leave a comment on youtube :)
+available. I think simply reducing LR by 1/10 or so, when loss plateaus is 
+a good default. 
 
 Programmed by Aladdin Persson <aladdin.persson at hotmail dot com>
 *    2020-04-10 Initial programming
+*    2022-12-19 Updated comments, made sure it works with latest PyTorch
 
 """
 
@@ -28,7 +27,9 @@
 
 # Hyperparameters
 num_classes = 10
-learning_rate = 0.1
+learning_rate = (
+    0.1  # way too high learning rate, but we want to see the scheduler in action
+)
 batch_size = 128
 num_epochs = 100
 
@@ -47,7 +48,7 @@
 
 # Define Scheduler
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
-    optimizer, factor=0.1, patience=5, verbose=True
+    optimizer, factor=0.1, patience=10, verbose=True
 )
 
 # Train Network
@@ -67,19 +68,19 @@
         losses.append(loss.item())
 
         # backward
+        optimizer.zero_grad()
         loss.backward()
-
-        # gradient descent or adam step
-        # scheduler.step(loss)
         optimizer.step()
-        optimizer.zero_grad()
 
     mean_loss = sum(losses) / len(losses)
+    mean_loss = round(mean_loss, 2)  # we should see difference in loss at 2 decimals
 
     # After each epoch do scheduler.step, note in this scheduler we need to send
-    # in loss for that epoch!
+    # in loss for that epoch! This can also be set using validation loss, and also
+    # in the forward loop we can do on our batch but then we might need to modify
+    # the patience parameter
     scheduler.step(mean_loss)
-    print(f"Cost at epoch {epoch} is {mean_loss}")
+    print(f"Average loss for epoch {epoch} was {mean_loss}")
 
 # Check accuracy on training & test to see how good our model
 def check_accuracy(loader, model):
@@ -90,6 +91,7 @@ def check_accuracy(loader, model):
     with torch.no_grad():
         for x, y in loader:
             x = x.to(device=device)
+            x = x.reshape(x.shape[0], -1)
             y = y.to(device=device)
 
             scores = model(x)

ML/Pytorch/Basics/pytorch_mixed_precision_example.py

@@ -34,7 +34,7 @@ def forward(self, x):
 # Hyperparameters
 in_channel = 1
 num_classes = 10
-learning_rate = 0.001
+learning_rate = 3e-4
 batch_size = 100
 num_epochs = 5
 
@@ -74,7 +74,6 @@ def forward(self, x):
 
 
 # Check accuracy on training & test to see how good our model
-
 def check_accuracy(loader, model):
     num_correct = 0
     num_samples = 0