Loss remains the same

[Aditi4AI]

torch.manual_seed(87)
epoch=80
for epochs in range(epoch):
model_0.train()
Y_pred=model_0(X_train)
loss=Loss(Y_pred,Y_train)
Optimizer.zero_grad()
loss.backward()
print(f"Loss:{loss}")
Optimizer.step()
model_0.eval()
with torch.inference_mode():
test_pred=model_0(X_test)
test_loss=Loss(test_pred,Y_test)
if epochs %2 ==0:
print (f"Epoch:{epochs}|Loss:{loss}|Test loss:{test_loss}")
with torch.inference_mode():
Y_preds_new=model_0(X_test)
plot_predictions(predictions=Y_preds_new);
print (model_0.state_dict())

I run this code,till 28th the output was descending i.e, going closer to zero :

Epoch:8|Loss:0.17129679024219513
Epoch:16|Loss:0.10746248811483383
Epoch:22|Loss:0.05994463711977005
Epoch:26|Loss:0.02823618240654468
Epoch:32|Loss:0.03788195922970772
Epoch:54|Loss:0.03788195922970772
Epoch:64|Loss:0.03788195922970772
Epoch:72|Loss:0.03788195922970772
Epoch:76|Loss:0.03788195922970772
Epoch:78|Loss:0.03788195922970772

But after 28th the loss remains same .WHY?

[Perian-Yan]

Hi,
I think the weights and biases are not updated, so the loss remains the same.

[Perian-Yan]

Are you doing linear regression? Did you visualize the data to see what's going on? Maybe you can try to change the hyperparameters like epochs and learning rate to see what happens.

[Perian-Yan]

Maybe you can send your model_0 architecture and the training/test set, so others can reproduce the result.

[Aditi4AI]

Yes! I have done.
Here's the code:
import numpy as np
torch.manual_seed(87)
epoch=100
for epochs in range(epoch):
model_0.train()
Y_pred=model_0(X_train)
loss=Loss(Y_pred,Y_train)
Optimizer.zero_grad()
loss.backward()
Optimizer.step()
model_0.eval()
with torch.inference_mode():
test_preds=model_0(X_test)
test_loss=Loss(test_preds,Y_test)
print (f"Epoch:{epochs}|Loss:{loss}|TestLoss:{test_loss}")
with torch.inference_mode():
Y_preds_new=model_0(X_test)
plot_predictions(predictions=Y_preds_new)

[Perian-Yan]

Hi,
your code seems to be correct, I run it and the result is reasonable. Since I don't know what is your model_0 and dataset, I just assume you're following the video and using a simple y = W*x+b linear model.

import torch
from torch import nn
import matplotlib.pyplot as plt
torch.__version__

device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")

weight = 0.7
bias = 0.3

start = 0
end = 1
step = 0.02
X = torch.arange(start, end, step).unsqueeze(dim=1)
y = weight * X + bias
print(f"\nthe head of X: {X[:10]}")
print(f"\nthe head of y: {y[:10]}")

train_split = int(0.8 * len(X))
X_train, y_train = X[:train_split], y[:train_split]
X_test, y_test = X[train_split:], y[train_split:]

# plot_predictions()
def plot_predictions(train_data=X_train,
                     train_labels=y_train,
                     test_data=X_test,
                     test_labels=y_test,
                     predictions=None):
  """
  Plots training data, test data and compares predictions.
  """
  plt.figure(figsize=(10, 7))

  # Plot training data in blue
  plt.scatter(train_data, train_labels, c="b", s=4, label="Training data")

  # Plot test data in green
  plt.scatter(test_data, test_labels, c="g", s=4, label="Testing data")

  if predictions is not None:
    # Plot the predictions in red (predictions were made on the test data)
    plt.scatter(test_data, predictions, c="r", s=4, label="Predictions")

  # Show the legend
  plt.legend(prop={"size": 14});  # show the labels on the image

class LinearRegressionModelV2(nn.Module):
  def __init__(self):
    super().__init__()
    # use nn.Linear()
    self.linear_layer = nn.Linear(in_features=1,
                                  out_features=1)
  def forward(self,x:torch.Tensor) -> torch.Tensor:
    return self.linear_layer(x)

model_1 = LinearRegressionModelV2()
model_1.state_dict()

model_1.to(device)
next(model_1.parameters()).device

loss_fn = nn.L1Loss()
optimizer = torch.optim.SGD(params=model_1.parameters(),
                            lr=0.01,
                            momentum=0.9)
X_train = X_train.to(device)
X_test = X_test.to(device)
y_train = y_train.to(device)
y_test = y_test.to(device)

########## basicly copy-paste from you #########
torch.manual_seed(87)
epoch=80
for epochs in range(epoch):
  model_1.train()
  y_pred=model_1(X_train)
  loss=loss_fn(y_pred,y_train)
  optimizer.zero_grad()
  loss.backward()
  optimizer.step()
  model_1.eval()
  with torch.inference_mode():
    test_pred=model_1(X_test)
    test_loss=loss_fn(test_pred,y_test)
  if epochs %4 ==0:
    print(f"Epoch:{epochs}|Loss:{loss}|Test loss:{test_loss}")

with torch.inference_mode():
  y_preds_new=model_1(X_test)

plot_predictions(predictions=y_preds_new.detach().cpu().numpy())
print (model_1.state_dict())
##############################################

The result is:

If you increase the epoch, say, 1000, the prediction will be better and the loss is also decreasing.

[Aditi4AI]

Yes, I followed the y=W*x+b linear model and my code is similar